def test_get_net_tests(self):
director = Director()
nettests = director.getNetTests()
assert "http_requests" in nettests
assert "dnsconsistency" in nettests
assert "http_header_field_manipulation" in nettests
assert "traceroute" in nettests
def test_get_net_tests(self):
director = Director()
nettests = director.getNetTests()
assert 'http_requests' in nettests
assert 'dns_consistency' in nettests
assert 'http_header_field_manipulation' in nettests
assert 'traceroute' in nettests
def test_singular_input_processor(self):
"""
Verify that all measurements use the same object as their input processor.
"""
ntl = NetTestLoader(dummyArgs)
ntl.loadNetTestString(generator_id_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.yamloo'
d = director.startNetTest(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
all_report_entries = yaml.safe_load_all(report_file)
header = all_report_entries.next()
results_case_a = all_report_entries.next()
aa_test, ab_test, ac_test = results_case_a.get('results', [])
results_case_b = all_report_entries.next()
ba_test = results_case_b.get('results', [])[0]
# Within a NetTestCase an inputs object will be consistent
self.assertEqual(aa_test, ab_test, ac_test)
# An inputs object will be different between different NetTestCases
self.assertNotEqual(aa_test, ba_test)
return d
def test_get_net_tests(self):
director = Director()
nettests = director.getNetTests()
assert 'http_requests' in nettests
assert 'dns_consistency' in nettests
assert 'http_header_field_manipulation' in nettests
assert 'traceroute' in nettests
def test_singular_input_processor(self):
"""
Verify that all measurements use the same object as their input processor.
"""
ntl = NetTestLoader(dummyArgs)
ntl.loadNetTestString(generator_id_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.yamloo'
d = director.startNetTest(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
all_report_entries = yaml.safe_load_all(report_file)
header = all_report_entries.next()
results_case_a = all_report_entries.next()
aa_test, ab_test, ac_test = results_case_a.get('results', [])
results_case_b = all_report_entries.next()
ba_test = results_case_b.get('results', [])[0]
# Within a NetTestCase an inputs object will be consistent
self.assertEqual(aa_test, ab_test, ac_test)
# An inputs object will be different between different NetTestCases
self.assertNotEqual(aa_test, ba_test)
return d
def runWithDirector(global_options, create_input_store=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
from ooni.director import Director
start_tor = False
director = Director()
if global_options['list']:
net_tests = [net_test for net_test in director.getNetTests().items()]
log.msg("")
log.msg("Installed nettests")
log.msg("==================")
for net_test_id, net_test in net_tests:
optList = []
for name, details in net_test['arguments'].items():
optList.append({'long': name, 'doc': details['description']})
desc = ('\n' +
net_test['name'] +
'\n' +
'-'*len(net_test['name']) +
'\n' +
'\n'.join(textwrap.wrap(net_test['description'], 80)) +
'\n\n' +
'$ ooniprobe {}/{}'.format(net_test['category'],
net_test['id']) +
'\n\n' +
''.join(usage.docMakeChunks(optList))
)
map(log.msg, desc.split("\n"))
log.msg("Note: Third party tests require an external "
"application to run properly.")
raise SystemExit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options.get('preferred-backend') is not None:
config.advanced.preferred_backend = global_options['preferred-backend']
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
elif config.advanced.get("preferred_backend", "onion") == "onion":
start_tor = True
if (global_options['collector'] and
config.advanced.get("preferred_backend", "onion") == "onion"):
start_tor |= True
return runTestWithDirector(
director=director,
start_tor=start_tor,
global_options=global_options,
create_input_store=create_input_store
)
def runWithDirector(global_options, create_input_store=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
from ooni.director import Director
start_tor = False
director = Director()
if global_options['list']:
net_tests = [net_test for net_test in director.getNetTests().items()]
log.msg("")
log.msg("Installed nettests")
log.msg("==================")
for net_test_id, net_test in net_tests:
optList = []
for name, details in net_test['arguments'].items():
optList.append({'long': name, 'doc': details['description']})
desc = ('\n' +
net_test['name'] +
'\n' +
'-'*len(net_test['name']) +
'\n' +
'\n'.join(textwrap.wrap(net_test['description'], 80)) +
'\n\n' +
'$ ooniprobe {}/{}'.format(net_test['category'],
net_test['id']) +
'\n\n' +
''.join(usage.docMakeChunks(optList))
)
map(log.msg, desc.split("\n"))
log.msg("Note: Third party tests require an external "
"application to run properly.")
raise SystemExit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options.get('preferred-backend') is not None:
config.advanced.preferred_backend = global_options['preferred-backend']
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
elif config.advanced.get("preferred_backend", "onion") == "onion":
start_tor = True
if (global_options['collector'] and
config.advanced.get("preferred_backend", "onion") == "onion"):
start_tor |= True
return runTestWithDirector(
director=director,
start_tor=start_tor,
global_options=global_options,
create_input_store=create_input_store
)
def runWithDirector():
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
# contains (test_cases, options, cmd_line_options)
test_list = []
director = Director()
d = director.start()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.netTests.items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
#XXX: This should mean no bouncer either!
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
global_options['bouncer'] = None
deck = Deck()
deck.bouncer = global_options['bouncer']
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'])
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def runWithDirector(global_options):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
from ooni.director import Director
director = Director()
if global_options['list']:
net_tests = [net_test for net_test in director.getNetTests().items()]
print ""
print "Installed nettests"
print "=================="
for net_test_id, net_test in net_tests:
optList = []
for name, details in net_test['arguments'].items():
optList.append({'long': name, 'doc': details['description']})
desc = ('\n' +
net_test['name'] +
'\n' +
'-'*len(net_test['name']) +
'\n' +
'\n'.join(textwrap.wrap(net_test['description'], 80)) +
'\n\n' +
'$ ooniprobe {}/{}'.format(net_test['category'],
net_test['id']) +
'\n\n' +
''.join(usage.docMakeChunks(optList))
)
print desc
print "Note: Third party tests require an external "\
"application to run properly."
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
if global_options['collector']:
start_tor |= True
return runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options)
def runWithDirector(global_options):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
from ooni.director import Director
start_tor = False
director = Director()
if global_options['list']:
net_tests = [net_test for net_test in director.getNetTests().items()]
print ""
print "Installed nettests"
print "=================="
for net_test_id, net_test in net_tests:
optList = []
for name, details in net_test['arguments'].items():
optList.append({'long': name, 'doc': details['description']})
desc = ('\n' + net_test['name'] + '\n' +
'-' * len(net_test['name']) + '\n' +
'\n'.join(textwrap.wrap(net_test['description'], 80)) +
'\n\n' + '$ ooniprobe {}/{}'.format(
net_test['category'], net_test['id']) + '\n\n' +
''.join(usage.docMakeChunks(optList)))
print desc
print "Note: Third party tests require an external "\
"application to run properly."
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options.get('preferred-backend') is not None:
config.advanced.preferred_backend = global_options['preferred-backend']
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
elif config.advanced.get("preferred_backend", "onion") == "onion":
start_tor = True
if (global_options['collector']
and config.advanced.get("preferred_backend", "onion") == "onion"):
start_tor |= True
return runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options)
def setUp(self):
self.director = Director()
self.testDetails = {'test_name': 'foo', 'start_time': time.time()}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
class TestStartSniffing(unittest.TestCase):
def setUp(self):
self.director = Director()
self.testDetails = {
'test_name': 'foo',
'test_start_time': '2016-01-01 12:34:56'
}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
def test_start_sniffing_once(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch(
'ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
self.director.startSniffing(self.testDetails)
sniffer = mock_scapy_sniffer.return_value
mock_scapy_factory.registerProtocol.assert_called_once_with(
sniffer)
def test_start_sniffing_twice(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch(
'ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
sniffer = mock_scapy_sniffer.return_value
sniffer.pcapwriter.filename = 'foo1_filename'
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 1)
self.testDetails = {
'test_name': 'bar',
'test_start_time': '2016-01-01 12:34:56'
}
with patch(
'ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
sniffer = mock_scapy_sniffer.return_value
sniffer.pcapwriter.filename = 'foo2_filename'
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 2)
def test_measurement_succeeded(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch(
'ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 1)
measurement = MagicMock()
measurement.testInstance = self.FooTestCase()
self.director.measurementSucceeded('awesome', measurement)
self.assertEqual(len(self.director.sniffers), 0)
sniffer = mock_scapy_sniffer.return_value
mock_scapy_factory.unRegisterProtocol.assert_called_once_with(
sniffer)
def setUp(self):
self.director = Director()
self.testDetails = {
'test_name': 'foo',
'test_start_time': '2016-01-01 12:34:56'
}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
def test_nettest_timeout(self):
ntl = NetTestLoader(('-u', 'http://localhost:8007/'))
ntl.loadNetTestString(http_net_test)
ntl.checkOptions()
director = Director()
d = director.startNetTest(ntl, [MockReporter()])
@d.addCallback
def complete(result):
assert director.failedMeasurements == 1
return d
class TestStartSniffing(unittest.TestCase):
def setUp(self):
self.director = Director()
self.testDetails = {
'test_name': 'foo',
'test_start_time': '2016-01-01 12:34:56'
}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
def test_start_sniffing_once(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
self.director.startSniffing(self.testDetails)
sniffer = mock_scapy_sniffer.return_value
mock_scapy_factory.registerProtocol.assert_called_once_with(sniffer)
def test_start_sniffing_twice(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
sniffer = mock_scapy_sniffer.return_value
sniffer.pcapwriter.filename = 'foo1_filename'
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 1)
self.testDetails = {
'test_name': 'bar',
'test_start_time': '2016-01-01 12:34:56'
}
with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
sniffer = mock_scapy_sniffer.return_value
sniffer.pcapwriter.filename = 'foo2_filename'
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 2)
def test_measurement_succeeded(self):
with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
self.director.startSniffing(self.testDetails)
self.assertEqual(len(self.director.sniffers), 1)
measurement = MagicMock()
measurement.testInstance = self.FooTestCase()
self.director.measurementSucceeded('awesome', measurement)
self.assertEqual(len(self.director.sniffers), 0)
sniffer = mock_scapy_sniffer.return_value
mock_scapy_factory.unRegisterProtocol.assert_called_once_with(sniffer)
def test_nettest_timeout(self):
ntl = NetTestLoader(('-u', 'http://localhost:8007/'))
ntl.loadNetTestString(http_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.yamloo'
d = director.startNetTest(ntl, self.filename)
@d.addCallback
def complete(result):
assert director.failedMeasurements == 1
return d
def test_nettest_timeout(self):
ntl = NetTestLoader(('-u', 'http://localhost:8007/'))
ntl.loadNetTestString(http_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.yamloo'
d = director.start_net_test_loader(ntl, self.filename)
@d.addCallback
def complete(result):
assert director.failedMeasurements == 1
return d
def test_net_test_completed_callback(self):
ntl = NetTestLoader(dummyArgsWithFile)
ntl.loadNetTestString(net_test_string_with_file)
ntl.checkOptions()
director = Director()
d = director.startNetTest(ntl, [MockReporter()])
@d.addCallback
def complete(result):
self.assertEqual(result, None)
self.assertEqual(director.successfulMeasurements, 20)
return d
def runWithDirector():
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
log.start(global_options["logfile"])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap")
sys.exit(2)
# contains (test_cases, options, cmd_line_options)
test_list = []
director = Director()
d = director.start()
# XXX: This should mean no bouncer either!
if global_options["no-collector"]:
log.msg("Not reporting using a collector")
collector = global_options["collector"] = None
global_options["bouncer"] = None
deck = Deck()
deck.bouncer = global_options["bouncer"]
try:
if global_options["testdeck"]:
deck.loadDeck(global_options["testdeck"])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options["test_file"])
net_test_loader = NetTestLoader(
global_options["subargs"], test_file=test_file, global_options=global_options
)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def setUp(self):
with open('dummyInputFile.txt', 'w') as f:
for i in range(10):
f.write("%s\n" % i)
self.reporters = [MockReporter()]
self.director = Director()
class TestDirector(unittest.TestCase):
timeout = 1
def setUp(self):
with open('dummyInputFile.txt', 'w') as f:
for i in range(10):
f.write("%s\n" % i)
self.reporters = [MockReporter()]
self.director = Director()
def tearDown(self):
pass
def test_start_net_test(self):
options = {'test':net_test_string, 'subargs':dummyArgs}
net_test_loader = NetTestLoader(options)
net_test_loader.checkOptions()
d = self.director.startNetTest('', net_test_loader, self.reporters)
@d.addCallback
def done(result):
self.assertEqual(self.director.successfulMeasurements, 20)
return d
def test_stop_net_test(self):
pass
def test_run_test_fails_twice(self):
finished = defer.Deferred()
def net_test_done(net_test):
summary_items = net_test.summary.items()
self.assertEqual(len(summary_items), 50)
for input_name, run_count in summary_items:
self.assertEqual(run_count, 3)
finished.callback(None)
net_test_loader = NetTestLoader(('spam', 'ham'))
net_test_loader.loadNetTestString(test_failing_twice)
director = Director()
director.netTestDone = net_test_done
director.startNetTest(net_test_loader, None, no_yamloo=True)
return finished
def test_net_test_completed_callback(self):
ntl = NetTestLoader(dummyArgsWithFile)
ntl.loadNetTestString(net_test_string_with_file)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.yamloo'
d = director.startNetTest(ntl, self.filename)
@d.addCallback
def complete(result):
self.assertEqual(result, None)
self.assertEqual(director.successfulMeasurements, 20)
return d
def test_run_test_fails_twice(self):
finished = defer.Deferred()
def net_test_done(net_test):
summary_items = net_test.summary.items()
self.assertEqual(len(summary_items), 50)
for input_name, run_count in summary_items:
self.assertEqual(run_count, 3)
finished.callback(None)
net_test_loader = NetTestLoader(('spam','ham'))
net_test_loader.loadNetTestString(test_failing_twice)
director = Director()
director.netTestDone = net_test_done
director.startNetTest(net_test_loader, None, no_yamloo=True)
return finished
def test_net_test_completed_callback(self):
ntl = NetTestLoader(dummyArgsWithFile)
ntl.loadNetTestString(net_test_string_with_file)
ntl.checkOptions()
director = Director()
d = director.startNetTest('', ntl, [MockReporter()])
@d.addCallback
def complete(result):
print "IN here y0"
self.assertEqual(result, None)
self.assertEqual(director.successfulMeasurements, 20)
return d
def test_net_test_completed_callback(self):
options = {'subargs':dummyArgsWithFile,
'test':StringIO(net_test_string_with_file)}
ntl = NetTestLoader(options)
ntl.checkOptions()
director = Director()
d = director.startNetTest('', ntl, [MockReporter()])
@d.addCallback
def complete(result):
#XXX: why is the return type (True, None) ?
self.assertEqual(result, [(True,None)])
self.assertEqual(director.successfulMeasurements, 20)
return d
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
if global_options['collector']:
start_tor |= True
return runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options,
check_incoherences=check_incoherences)
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'], net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
if global_options['collector']:
start_tor |= True
return runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options,
check_incoherences=check_incoherences)
def test_send_to_inputs_generator(self):
"""
Verify that a net test can send information back into an inputs generator.
"""
ntl = NetTestLoader(dummyArgs)
ntl.loadNetTestString(generator_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.njson'
d = director.start_net_test_loader(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
all_report_entries = map(json.loads, report_file)
results = [x['input'] for x in all_report_entries]
self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0])
return d
def test_send_to_inputs_generator(self):
"""
Verify that a net test can send information back into an inputs generator.
"""
ntl = NetTestLoader(dummyArgs)
ntl.loadNetTestString(generator_net_test)
ntl.checkOptions()
director = Director()
self.filename = 'dummy_report.njson'
d = director.start_net_test_loader(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
all_report_entries = map(json.loads, report_file)
results = [x['input'] for x in all_report_entries]
self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0])
return d
def setUp(self):
self.director = Director()
self.testDetails = {
'test_name': 'foo',
'test_start_time': '2016-01-01 12:34:56'
}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
def setUp(self):
self.director = Director()
self.testDetails = {
'test_name': 'foo',
'start_time': time.time()
}
# Each NetTestCase has a name attribute
class FooTestCase(object):
name = 'foo'
self.FooTestCase = FooTestCase
def __init__(self, web_ui_port):
"""
If the advanced->disabled_webui is set to true then we will not start the WebUI.
"""
service.MultiService.__init__(self)
director = Director()
self.scheduler_service = SchedulerService(director)
self.scheduler_service.setServiceParent(self)
if not config.advanced.disabled_webui:
self.web_ui_service = WebUIService(director,
self.scheduler_service,
web_ui_port)
self.web_ui_service.setServiceParent(self)
def getOonid():
director = Director()
director.start()
oonidApplication.director = director
return internet.TCPServer(int(config.advanced.oonid_api_port),
oonidApplication)
def director_start_tor():
director = Director()
yield director.startTor()
assert config.tor.socks_port == 4242
assert config.tor.control_port == 4242
def director_start_tor():
self.config.advanced.start_tor = True
director = Director()
yield director.start_tor()
self.assertEqual(config.tor.socks_port, 4242)
self.assertEqual(config.tor.control_port, 4242)
def runWithDaemonDirector(global_options):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
from twisted.internet import reactor, protocol
from ooni.director import Director
try:
import pika
from pika import exceptions
from pika.adapters import twisted_connection
except ImportError:
print "Pika is required for queue connection."
print "Install with \"pip install pika\"."
sys.exit(7)
director = Director()
if global_options.get('annotations') is not None:
global_options['annotations'] = setupAnnotations(global_options)
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
finished = defer.Deferred()
@defer.inlineCallbacks
def readmsg(_, channel, queue_object, consumer_tag, counter):
# Wait for a message and decode it.
if counter >= lifetime:
log.msg("Counter")
queue_object.close(LifetimeExceeded())
yield channel.basic_cancel(consumer_tag=consumer_tag)
finished.callback(None)
else:
log.msg("Waiting for message")
try:
ch, method, properties, body = yield queue_object.get()
log.msg("Got message")
data = json.loads(body)
counter += 1
log.msg("Received %d/%d: %s" % (
counter,
lifetime,
data['url'],
))
# acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
d = runTestWithDirector(director=director,
start_tor=start_tor,
global_options=global_options,
url=data['url'].encode('utf8'))
# When the test has been completed, go back to waiting for a message.
d.addCallback(readmsg, channel, queue_object, consumer_tag,
counter + 1)
except exceptions.AMQPError, v:
log.msg("Error")
log.exception(v)
finished.errback(v)
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'], net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector, errors.ProbeIPUnknown,
errors.InvalidInputFile, errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n"
)
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues"
)
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly."
)
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n."
)
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'], collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
def getOonid():
director = Director()
director.start()
oonidApplication.director = director
return internet.TCPServer(int(config.advanced.oonid_api_port), oonidApplication)
test_file=global_options['test_file']))
# check each test's usageOptions
for net_test_loader in test_list:
try:
net_test_loader.checkOptions()
except MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
except usage.UsageError, e:
log.err(e)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
director = Director()
d = director.start()
def director_startup_failed(failure):
log.err("Failed to start the director")
r = failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress)
if r == errors.TorNotRunning:
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n"
)
elif r == errors.InvalidOONIBCollectorAddress:
log.err("Invalid format for oonib collector address.")
test_file=global_options['test_file']))
# check each test's usageOptions
for net_test_loader in test_list:
try:
net_test_loader.checkOptions()
except MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
except usage.UsageError, e:
log.err(e)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
director = Director()
d = director.start()
def director_startup_failed(failure):
log.err("Failed to start the director")
r = failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress)
if r == errors.TorNotRunning:
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg("Try with a different collector or disable collector reporting with -n")
elif r == errors.InvalidOONIBCollectorAddress:
log.err("Invalid format for oonib collector address.")
log.msg("Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector,
errors.ProbeIPUnknown,
errors.InvalidInputFile,
errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n")
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly.")
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'],
collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
def runWithDirector(logging=True, start_tor=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
#XXX: This should mean no bouncer either!
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
global_options['bouncer'] = None
deck = Deck()
deck.bouncer = global_options['bouncer']
start_tor = deck.requiresTor
if global_options['bouncer']:
start_tor = True
if global_options['collector']:
start_tor = True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'])
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def director_start_tor():
director = Director()
yield director.startTor()
assert config.tor.socks_port == 4242
assert config.tor.control_port == 4242
