def createDeck(global_options, url=None):
from ooni.nettest import NetTestLoader
from ooni.deck import Deck, nettest_to_path
if url:
log.msg("Creating deck for: %s" % (url))
if global_options['no-yamloo']:
log.msg("Will not write to a yamloo report file")
deck = Deck(bouncer=global_options['bouncer'],
no_collector=global_options['no-collector'])
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'], global_options)
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
net_test_loader = NetTestLoader(args,
test_file=test_file,
annotations=global_options['annotations'])
if global_options['collector']:
net_test_loader.collector = \
CollectorClient(global_options['collector'])
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except errors.HTTPSCollectorUnsupported:
log.err("HTTPS collectors require a twisted version of at least 14.0.2.")
sys.exit(6)
except errors.InsecureBackend:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_backend option to true in "
"your ooniprobe.conf file.")
sys.exit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
return deck
def test_save_deck_descriptor(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.loadDeck(self.deck_file)
deck.load(
{"name": "spam", "id": "spam", "version": "spam", "author": "spam", "date": "spam", "description": "spam"}
)
deck.save()
deck.verify()
def test_lookuptest_helpers(self):
deck = Deck(deckFile=self.deck_file, decks_directory=".")
deck.oonibclient = MockOONIBClient()
yield deck.lookupTestHelpers()
assert deck.netTestLoaders[0].collector == 'httpo://thirteenchars1234.onion'
required_test_helpers = deck.netTestLoaders[0].requiredTestHelpers
assert len(required_test_helpers) == 1
assert required_test_helpers[0]['test_class'].localOptions['spam'] == '127.0.0.1'
def test_deck_with_many_tests(self):
os.remove(self.deck_file)
deck_hash = sha256(self.dummy_deck_content_with_many_tests).hexdigest()
self.deck_file = os.path.join(self.cwd, deck_hash)
with open(self.deck_file, 'w+') as f:
f.write(self.dummy_deck_content_with_many_tests)
deck = Deck(decks_directory=".")
deck.loadDeck(self.deck_file)
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'1.1.1.1')
self.assertEqual(deck.netTestLoaders[1].localOptions['backend'],
'2.2.2.2')
def test_load_deck_with_global_options(self):
global_options = {
"annotations": {
"spam": "ham"
},
"collector": "httpo://thirteenchars123.onion"
}
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".")
deck.loadDeck(self.deck_file, global_options=global_options)
self.assertEqual(deck.netTestLoaders[0].annotations,
global_options['annotations'])
self.assertEqual(
deck.netTestLoaders[0].collector.base_address,
global_options['collector'].replace("httpo://", "http://"))
def test_save_deck_descriptor(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".")
deck.loadDeck(self.deck_file)
deck.load({
'name': 'spam',
'id': 'spam',
'version': 'spam',
'author': 'spam',
'date': 'spam',
'description': 'spam'
})
deck.save()
self.filename = self.deck_file + ".desc"
deck.verify()
def test_save_deck_descriptor(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.loadDeck(self.deck_file)
deck.load({'name': 'spam',
'id': 'spam',
'version': 'spam',
'author': 'spam',
'date': 'spam',
'description': 'spam'
})
deck.save()
self.filename = self.deck_file + ".desc"
deck.verify()
def test_load_deck_with_global_options(self):
global_options = {
"annotations": {"spam": "ham"},
"collector": "httpo://thirteenchars123.onion"
}
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file,
global_options=global_options)
self.assertEqual(
deck.netTestLoaders[0].annotations,
global_options['annotations']
)
self.assertEqual(
deck.netTestLoaders[0].collector.base_address,
global_options['collector'].replace("httpo://", "http://")
)
def test_deck_with_many_tests(self):
os.remove(self.deck_file)
deck_hash = sha256(self.dummy_deck_content_with_many_tests).hexdigest()
self.deck_file = os.path.join(self.cwd, deck_hash)
with open(self.deck_file, 'w+') as f:
f.write(self.dummy_deck_content_with_many_tests)
deck = Deck(decks_directory=".")
deck.loadDeck(self.deck_file)
self.assertEqual(
deck.netTestLoaders[0].localOptions['backend'],
'1.1.1.1'
)
self.assertEqual(
deck.netTestLoaders[1].localOptions['backend'],
'2.2.2.2'
)
def test_save_deck_descriptor(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file)
deck.load({'name': 'spam',
'id': 'spam',
'version': 'spam',
'author': 'spam',
'date': 'spam',
'description': 'spam'
})
deck.save()
self.filename = self.deck_file + ".desc"
deck.verify()
def test_lookup_test_helpers_and_collector_cloudfront(self):
self.config.advanced.preferred_backend = "cloudfront"
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
deck._BouncerClient = MockBouncerClient
deck._CollectorClient = MockCollectorClient
deck.loadDeck(self.deck_file)
self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
yield deck.lookupCollectorAndTestHelpers()
self.assertEqual(
deck.netTestLoaders[0].collector.settings['address'],
'https://address.cloudfront.net'
)
self.assertEqual(
deck.netTestLoaders[0].collector.settings['front'],
'front.cloudfront.net'
)
self.assertEqual(
deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1'
)
def createDeck(global_options, url=None):
if url:
log.msg("Creating deck for: %s" % (url))
if global_options['no-yamloo']:
log.msg("Will not write to a yamloo report file")
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
if url is not None:
args = ('-u', url)
else:
args = tuple()
if any(global_options['subargs']):
args = global_options['subargs'] + args
net_test_loader = NetTestLoader(args, test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except errors.HTTPSCollectorUnsupported:
log.err(
"HTTPS collectors require a twisted version of at least 14.0.2.")
sys.exit(6)
except errors.InsecureCollector:
log.err("Attempting to report to an insecure collector.")
log.err("To enable reporting to insecure collector set the "
"advanced->insecure_collector option to true in "
"your ooniprobe.conf file.")
sys.exit(7)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
return deck
def test_save_deck_descriptor(self):
deck = Deck(deckFile=self.deck_file, decks_directory=".")
deck.load({'name': 'spam',
'id': 'spam',
'version': 'spam',
'author': 'spam',
'date': 'spam',
'description': 'spam'
})
deck.save()
deck.verify()
def test_lookuptest_helpers(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.oonibclient = MockOONIBClient()
deck.loadDeck(self.deck_file)
yield deck.lookupTestHelpers()
assert deck.netTestLoaders[0].collector == 'httpo://thirteenchars1234.onion'
required_test_helpers = deck.netTestLoaders[0].requiredTestHelpers
assert len(required_test_helpers) == 1
assert required_test_helpers[0]['test_class'].localOptions['backend'] == '127.0.0.1'
def runWithDirector():
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
# contains (test_cases, options, cmd_line_options)
test_list = []
director = Director()
d = director.start()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.netTests.items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
#XXX: This should mean no bouncer either!
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
global_options['bouncer'] = None
deck = Deck()
deck.bouncer = global_options['bouncer']
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'])
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def test_lookup_test_helpers_and_collector(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".")
deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
deck._BouncerClient = MockBouncerClient
deck._CollectorClient = MockCollectorClient
deck.loadDeck(self.deck_file)
self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
yield deck.lookupCollectorAndTestHelpers()
self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
'httpo://thirteenchars123.onion')
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1')
def test_lookup_test_helpers_and_collector_https(self):
self.config.advanced.preferred_backend = "https"
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".")
deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
deck._BouncerClient = MockBouncerClient
deck._CollectorClient = MockCollectorClient
deck.loadDeck(self.deck_file)
self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
yield deck.lookupCollectorAndTestHelpers()
self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
'https://collector.ooni.io')
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1')
def test_lookuptest_helpers(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.oonibclient = MockOONIBClient()
deck.loadDeck(self.deck_file)
yield deck.lookupTestHelpers()
assert deck.netTestLoaders[0].collector == "httpo://thirteenchars1234.onion"
required_test_helpers = deck.netTestLoaders[0].requiredTestHelpers
assert len(required_test_helpers) == 1
assert required_test_helpers[0]["test_class"].localOptions["backend"] == "127.0.0.1"
def test_lookup_test_helpers_and_collector(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
deck._BouncerClient = MockBouncerClient
deck._CollectorClient = MockCollectorClient
deck.loadDeck(self.deck_file)
self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
yield deck.lookupCollectorAndTestHelpers()
self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
'httpo://thirteenchars123.onion')
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1')
def downloadDeck(self, deck_hash):
from ooni.deck import Deck
deck = Deck(deck_hash)
if deck.fileCached:
return defer.succeed(deck)
else:
d = self.download('/deck/' + deck_hash + '/file', deck.cached_file)
@d.addCallback
def cb(res):
deck.verify()
return deck
@d.addErrback
def err(err):
log.err("Failed to download the deck %s" % deck_hash)
log.exception(err)
return d
def test_lookuptest_helpers(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.oonibclient = MockOONIBClient()
deck.loadDeck(self.deck_file)
self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
yield deck.lookupTestHelpers()
self.assertEqual(deck.netTestLoaders[0].collector,
'httpo://thirteenchars1234.onion')
self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
'127.0.0.1')
def runWithDirector():
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
log.start(global_options["logfile"])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap")
sys.exit(2)
# contains (test_cases, options, cmd_line_options)
test_list = []
director = Director()
d = director.start()
# XXX: This should mean no bouncer either!
if global_options["no-collector"]:
log.msg("Not reporting using a collector")
collector = global_options["collector"] = None
global_options["bouncer"] = None
deck = Deck()
deck.bouncer = global_options["bouncer"]
try:
if global_options["testdeck"]:
deck.loadDeck(global_options["testdeck"])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options["test_file"])
net_test_loader = NetTestLoader(
global_options["subargs"], test_file=test_file, global_options=global_options
)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def getDeck(self, deck_hash):
from ooni.deck import Deck
deck = Deck(deck_hash)
if deck.descriptorCached:
return defer.succeed(deck)
else:
d = self.queryBackend('GET', '/deck/' + deck_hash)
@d.addCallback
def cb(descriptor):
deck.load(descriptor)
deck.save()
return deck
@d.addErrback
def err(err):
log.err("Failed to get descriptor for deck %s" % deck_hash)
log.exception(err)
return d
def test_open_deck(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS, decks_directory=".")
deck.loadDeck(self.deck_file)
assert len(deck.netTestLoaders) == 1
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
if hasRawSocketPermission():
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
else:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'], net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector, errors.ProbeIPUnknown,
errors.InvalidInputFile, errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n"
)
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues"
)
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly."
)
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n."
)
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'], collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
def test_open_deck(self):
deck = Deck(bouncer="httpo://foo.onion",
decks_directory=".")
deck.loadDeck(self.deck_file)
assert len(deck.netTestLoaders) == 1
def test_save_deck_descriptor(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.loadDeck(self.deck_file)
deck.load({'name': 'spam',
'id': 'spam',
'version': 'spam',
'author': 'spam',
'date': 'spam',
'description': 'spam'
})
deck.save()
self.filename = self.deck_file + ".desc"
deck.verify()
def test_open_deck(self):
deck = Deck(decks_directory=".")
deck.bouncer = "httpo://foo.onion"
deck.loadDeck(self.deck_file)
assert len(deck.netTestLoaders) == 1
def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.initialize_ooni_home()
try:
config.read_config_file(check_incoherences=check_incoherences)
except errors.ConfigFileIncoherent:
sys.exit(6)
if global_options['verbose']:
config.advanced.debug = True
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
from ooni.utils.txscapy import ScapyFactory
config.scapyFactory = ScapyFactory(config.advanced.interface)
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
if global_options.get('annotations') is not None:
annotations = {}
for annotation in global_options["annotations"].split(","):
pair = annotation.split(":")
if len(pair) == 2:
key = pair[0].strip()
value = pair[1].strip()
annotations[key] = value
else:
log.err("Invalid annotation: %s" % annotation)
sys.exit(1)
global_options["annotations"] = annotations
if global_options['no-collector']:
log.msg("Not reporting using a collector")
global_options['collector'] = None
start_tor = False
else:
start_tor = True
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
if global_options['collector']:
start_tor |= True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
deck.insert(net_test_loader)
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
print incomplete_net_test_loader.usageOptions().getUsage()
sys.exit(2)
except errors.NetTestNotFound as path:
log.err('Requested NetTest file not found (%s)' % path)
sys.exit(3)
except errors.OONIUsageError as e:
log.err(e)
print e.net_test_loader.usageOptions().getUsage()
sys.exit(4)
except Exception as e:
if config.advanced.debug:
log.exception(e)
log.err(e)
sys.exit(5)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
check_incoherences=check_incoherences)
def setup_nettest(_):
try:
return deck.setup()
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
def director_startup_handled_failures(failure):
log.err("Could not start the director")
failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
errors.UnableToLoadDeckInput,
errors.CouldNotFindTestHelper,
errors.CouldNotFindTestCollector,
errors.ProbeIPUnknown,
errors.InvalidInputFile,
errors.ConfigFileIncoherent)
if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg(
"Try with a different collector or disable collector reporting with -n")
elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg(
"Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
elif isinstance(failure.value, errors.UnableToLoadDeckInput):
log.err("Unable to fetch the required inputs for the test deck.")
log.msg(
"Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
elif isinstance(failure.value, errors.CouldNotFindTestHelper):
log.err("Unable to obtain the required test helpers.")
log.msg(
"Try with a different bouncer or check that Tor is running properly.")
elif isinstance(failure.value, errors.CouldNotFindTestCollector):
log.err("Could not find a valid collector.")
log.msg(
"Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
elif isinstance(failure.value, errors.ProbeIPUnknown):
log.err("Failed to lookup probe IP address.")
log.msg("Check your internet connection.")
elif isinstance(failure.value, errors.InvalidInputFile):
log.err("Invalid input file \"%s\"" % failure.value)
elif isinstance(failure.value, errors.ConfigFileIncoherent):
log.err("Incoherent config file")
if config.advanced.debug:
log.exception(failure)
def director_startup_other_failures(failure):
log.err("An unhandled exception occurred while starting the director!")
log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
for net_test_loader in deck.netTestLoaders:
# Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
# specifying a collector from the command-line (-c).
# If a collector is not specified in the deck, or the
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
collector = None
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
elif 'collector' in config.reports \
and config.reports['collector']:
collector = config.reports['collector']
elif net_test_loader.collector:
collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
director.startNetTest(net_test_loader,
global_options['reportfile'],
collector)
return director.allTestsDone
def start():
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
return start()
def runWithDirector(logging=True, start_tor=True):
"""
Instance the director, parse command line options and start an ooniprobe
test!
"""
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
config.read_config_file()
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
if config.privacy.includepcap:
try:
checkForRoot()
except errors.InsufficientPrivileges:
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
director = Director()
if global_options['list']:
print "# Installed nettests"
for net_test_id, net_test in director.getNetTests().items():
print "* %s (%s/%s)" % (net_test['name'],
net_test['category'],
net_test['id'])
print " %s" % net_test['description']
sys.exit(0)
elif global_options['printdeck']:
del global_options['printdeck']
print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
print yaml.safe_dump([{'options': global_options}]).strip()
sys.exit(0)
#XXX: This should mean no bouncer either!
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
global_options['bouncer'] = None
deck = Deck()
deck.bouncer = global_options['bouncer']
start_tor = deck.requiresTor
if global_options['bouncer']:
start_tor = True
if global_options['collector']:
start_tor = True
try:
if global_options['testdeck']:
deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'])
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=test_file)
deck.insert(net_test_loader)
except errors.MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
def test_open_deck(self):
deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
decks_directory=".")
deck.loadDeck(self.deck_file)
assert len(deck.netTestLoaders) == 1
