def loadDeck(self, deckFile, global_options={}): with open(deckFile) as f: self.id = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) for test in test_deck: try: nettest_path = nettest_to_path(test['options']['test_file']) except e.NetTestNotFound: log.err("Could not find %s" % test['options']['test_file']) log.msg("Skipping...") continue annotations = test['options'].get('annotations', {}) if global_options.get('annotations') is not None: annotations = global_options["annotations"] collector_address = test['options'].get('collector', None) if global_options.get('collector') is not None: collector_address = global_options['collector'] net_test_loader = NetTestLoader(test['options']['subargs'], annotations=annotations, test_file=nettest_path) if collector_address is not None: net_test_loader.collector = CollectorClient(collector_address) if test['options'].get('bouncer', None) is not None: self.bouncer = self._BouncerClient(test['options']['bouncer']) if self.bouncer.backend_type == "onion": self.requiresTor = True self.insert(net_test_loader)
def test_singular_input_processor(self): """ Verify that all measurements use the same object as their input processor. """ ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(generator_id_net_test) ntl.checkOptions() director = Director() self.filename = 'dummy_report.yamloo' d = director.startNetTest(ntl, self.filename) @d.addCallback def complete(result): with open(self.filename) as report_file: all_report_entries = yaml.safe_load_all(report_file) header = all_report_entries.next() results_case_a = all_report_entries.next() aa_test, ab_test, ac_test = results_case_a.get('results', []) results_case_b = all_report_entries.next() ba_test = results_case_b.get('results', [])[0] # Within a NetTestCase an inputs object will be consistent self.assertEqual(aa_test, ab_test, ac_test) # An inputs object will be different between different NetTestCases self.assertNotEqual(aa_test, ba_test) return d
def test_load_with_missing_required_option(self): try: ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string_with_required_option) except MissingRequiredOption: pass
def loadDeck(self, deckFile, global_options={}): with open(deckFile) as f: self.id = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) for test in test_deck: try: nettest_path = nettest_to_path(test['options']['test_file']) except e.NetTestNotFound: log.err("Could not find %s" % test['options']['test_file']) log.msg("Skipping...") continue annotations = test['options'].get('annotations', {}) if global_options.get('annotations') is not None: annotations = global_options["annotations"] collector_address = test['options'].get('collector', None) if global_options.get('collector') is not None: collector_address = global_options['collector'] net_test_loader = NetTestLoader(test['options']['subargs'], annotations=annotations, test_file=nettest_path) if collector_address is not None: net_test_loader.collector = CollectorClient( collector_address ) if test['options'].get('bouncer', None) is not None: self.bouncer = self._BouncerClient(test['options']['bouncer']) if self.bouncer.backend_type is "onion": self.requiresTor = True self.insert(net_test_loader)
def test_setup_local_options_in_test_cases(self): options = {'subargs':dummyArgs, 'test':StringIO(net_test_string)} ntl = NetTestLoader(options) ntl.checkOptions() for test_class, test_method in ntl.testCases: self.assertEqual(test_class.localOptions, dummyOptions)
def createDeck(global_options, url=None): from ooni.nettest import NetTestLoader from ooni.deck import Deck, nettest_to_path if url: log.msg("Creating deck for: %s" % (url)) if global_options['no-yamloo']: log.msg("Will not write to a yamloo report file") deck = Deck(bouncer=global_options['bouncer'], no_collector=global_options['no-collector']) try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck'], global_options) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file'], True) if url is not None: args = ('-u', url) else: args = tuple() if any(global_options['subargs']): args = global_options['subargs'] + args net_test_loader = NetTestLoader(args, test_file=test_file, annotations=global_options['annotations']) if global_options['collector']: net_test_loader.collector = \ CollectorClient(global_options['collector']) deck.insert(net_test_loader) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() sys.exit(2) except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) sys.exit(3) except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) except errors.HTTPSCollectorUnsupported: log.err("HTTPS collectors require a twisted version of at least 14.0.2.") sys.exit(6) except errors.InsecureBackend: log.err("Attempting to report to an insecure collector.") log.err("To enable reporting to insecure collector set the " "advanced->insecure_backend option to true in " "your ooniprobe.conf file.") sys.exit(7) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) return deck
def test_load_net_test_multiple_different_options(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(double_different_options_net_test_string) test_cases = ntl.getTestCases() self.verifyMethods(test_cases) self.assertRaises(IncoherentOptions, ntl.checkOptions)
def test_require_root_succeed(self): # XXX: will require root to run ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_root_required) for test_class, methods in ntl.getTestCases(): self.assertTrue(test_class.requiresRoot)
def test_require_root_succeed(self): #XXX: will require root to run ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_root_required) for test_class, method in ntl.testCases: self.assertTrue(test_class.requiresRoot)
def test_load_net_test_multiple_different_options(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(double_different_options_net_test_string) self.verifyMethods(ntl.testCases) self.verifyClasses(ntl.testCases, set(('DummyTestCaseA', 'DummyTestCaseB'))) self.assertRaises(IncoherentOptions, ntl.checkOptions)
def test_load_with_option(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) self.assertIsInstance(ntl, NetTestLoader) for test_klass, test_meth in ntl.getTestCases(): for option in dummyOptions.keys(): self.assertIn(option, test_klass.usageOptions())
def test_load_with_option(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) self.assertIsInstance(ntl, NetTestLoader) for test_klass, test_meth in ntl.testCases: for option in dummyOptions.keys(): self.assertIn(option, test_klass.usageOptions())
def test_setup_local_options_in_test_cases(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) ntl.checkOptions() for test_class, test_method in ntl.testCases: self.assertEqual(test_class.localOptions, dummyOptions)
def runWithDirector(): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() config.global_options = global_options config.set_paths() config.read_config_file() log.start(global_options['logfile']) if config.privacy.includepcap: try: checkForRoot() except errors.InsufficientPrivileges: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) # contains (test_cases, options, cmd_line_options) test_list = [] director = Director() d = director.start() if global_options['list']: print "# Installed nettests" for net_test_id, net_test in director.netTests.items(): print "* %s (%s/%s)" % (net_test['name'], net_test['category'], net_test['id']) print " %s" % net_test['description'] sys.exit(0) #XXX: This should mean no bouncer either! if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None global_options['bouncer'] = None deck = Deck() deck.bouncer = global_options['bouncer'] try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file']) net_test_loader = NetTestLoader(global_options['subargs'], test_file=test_file) deck.insert(net_test_loader) except errors.MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name) print net_test_loader.usageOptions().getUsage() sys.exit(2)
def test_load_net_test_from_StringIO(self): """ Given a file like object verify that the net test cases are properly generated. """ ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) self.verifyMethods(ntl.testCases)
def test_generate_measurements_size(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() net_test = NetTest(ntl, None) measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)
def test_load_with_invalid_option(self): options = {'subargs':dummyInvalidArgs, 'test':StringIO(net_test_string)} try: ntl = NetTestLoader(options) ntl.checkOptions() raise Exception except UsageError: pass
def test_load_net_test_from_str(self): """ Given a file like object verify that the net test cases are properly generated. """ ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) self.verifyMethods(ntl.getTestCases())
def test_load_with_invalid_option(self): try: ntl = NetTestLoader(dummyInvalidArgs) ntl.loadNetTestString(net_test_string) ntl.checkOptions() raise Exception except UsageError: pass
def test_generate_measurements_size(self): options = {'subargs':dummyArgsWithFile, 'test':StringIO(net_test_string_with_file)} ntl = NetTestLoader(options) ntl.checkOptions() net_test = NetTest(ntl, None) measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)
def createDeck(global_options, url=None): if url: log.msg("Creating deck for: %s" % (url)) if global_options['no-yamloo']: log.msg("Will not write to a yamloo report file") deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file'], True) if url is not None: args = ('-u', url) else: args = tuple() if any(global_options['subargs']): args = global_options['subargs'] + args net_test_loader = NetTestLoader(args, test_file=test_file) if global_options['collector']: net_test_loader.collector = global_options['collector'] deck.insert(net_test_loader) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() sys.exit(2) except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) sys.exit(3) except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) except errors.HTTPSCollectorUnsupported: log.err( "HTTPS collectors require a twisted version of at least 14.0.2.") sys.exit(6) except errors.InsecureCollector: log.err("Attempting to report to an insecure collector.") log.err("To enable reporting to insecure collector set the " "advanced->insecure_collector option to true in " "your ooniprobe.conf file.") sys.exit(7) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) return deck
def test_start_net_test(self): options = {'test':net_test_string, 'subargs':dummyArgs} net_test_loader = NetTestLoader(options) net_test_loader.checkOptions() d = self.director.startNetTest('', net_test_loader, self.reporters) @d.addCallback def done(result): self.assertEqual(self.director.successfulMeasurements, 20) return d
def test_net_test_inputs(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() # XXX: if you use the same test_class twice you will have consumed all # of its inputs! tested = set([]) for test_class, test_method in ntl.testCases: if test_class not in tested: tested.update([test_class]) self.assertEqual(len(list(test_class.inputs)), 10)
def _load_ooni(self, task_data): required_keys = ["test_name"] for required_key in required_keys: if required_key not in task_data: raise MissingTaskDataKey(required_key) self.ooni['test_name'] = task_data.pop('test_name') # This raises e.NetTestNotFound, we let it go onto the caller nettest_path = nettest_to_path(self.ooni['test_name'], self._arbitrary_paths) annotations = self._pop_option('annotations', task_data, {}) collector_address = self._pop_option('collector', task_data, None) try: self.output_path = self.global_options['reportfile'] except KeyError: self.output_path = task_data.pop('reportfile', None) if task_data.get('no-collector', False): collector_address = None elif config.reports.upload is False: collector_address = None net_test_loader = NetTestLoader( options_to_args(task_data), annotations=annotations, test_file=nettest_path ) if isinstance(collector_address, dict): net_test_loader.collector = CollectorClient( settings=collector_address ) elif collector_address is not None: net_test_loader.collector = CollectorClient( collector_address ) if (net_test_loader.collector is not None and net_test_loader.collector.backend_type == "onion"): self.requires_tor = True try: net_test_loader.checkOptions() if net_test_loader.requiresTor: self.requires_tor = True except e.MissingTestHelper: self.requires_bouncer = True self.ooni['net_test_loader'] = net_test_loader
def test_nettest_timeout(self): ntl = NetTestLoader(('-u', 'http://localhost:8007/')) ntl.loadNetTestString(http_net_test) ntl.checkOptions() director = Director() d = director.startNetTest(ntl, [MockReporter()]) @d.addCallback def complete(result): assert director.failedMeasurements == 1 return d
def test_load_net_test_from_file(self): """ Given a file verify that the net test cases are properly generated. """ __, net_test_file = mkstemp() with open(net_test_file, 'w') as f: f.write(net_test_string) f.close() ntl = NetTestLoader(dummyArgs) ntl.loadNetTestFile(net_test_file) self.verifyMethods(ntl.getTestCases()) os.unlink(net_test_file)
def test_load_net_test_from_file(self): """ Given a file verify that the net test cases are properly generated. """ __, net_test_file = mkstemp() with open(net_test_file, 'w') as f: f.write(net_test_string) f.close() ntl = NetTestLoader(dummyArgs) ntl.loadNetTestFile(net_test_file) self.verifyMethods(ntl.testCases) os.unlink(net_test_file)
def test_nettest_timeout(self): ntl = NetTestLoader(('-u', 'http://localhost:8007/')) ntl.loadNetTestString(http_net_test) ntl.checkOptions() director = Director() self.filename = 'dummy_report.yamloo' d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): assert director.failedMeasurements == 1 return d
def test_net_test_completed_callback(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() director = Director() d = director.startNetTest(ntl, [MockReporter()]) @d.addCallback def complete(result): self.assertEqual(result, None) self.assertEqual(director.successfulMeasurements, 20) return d
def runWithDirector(): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() config.global_options = global_options config.set_paths() config.read_config_file() log.start(global_options["logfile"]) if config.privacy.includepcap: try: checkForRoot() except errors.InsufficientPrivileges: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) # contains (test_cases, options, cmd_line_options) test_list = [] director = Director() d = director.start() # XXX: This should mean no bouncer either! if global_options["no-collector"]: log.msg("Not reporting using a collector") collector = global_options["collector"] = None global_options["bouncer"] = None deck = Deck() deck.bouncer = global_options["bouncer"] try: if global_options["testdeck"]: deck.loadDeck(global_options["testdeck"]) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options["test_file"]) net_test_loader = NetTestLoader( global_options["subargs"], test_file=test_file, global_options=global_options ) deck.insert(net_test_loader) except errors.MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name) print net_test_loader.usageOptions().getUsage() sys.exit(2)
def test_net_test_completed_callback(self): options = {'subargs':dummyArgsWithFile, 'test':StringIO(net_test_string_with_file)} ntl = NetTestLoader(options) ntl.checkOptions() director = Director() d = director.startNetTest('', ntl, [MockReporter()]) @d.addCallback def complete(result): #XXX: why is the return type (True, None) ? self.assertEqual(result, [(True,None)]) self.assertEqual(director.successfulMeasurements, 20) return d
def test_run_test_fails_twice(self): finished = defer.Deferred() def net_test_done(net_test): summary_items = net_test.summary.items() self.assertEqual(len(summary_items), 50) for input_name, run_count in summary_items: self.assertEqual(run_count, 3) finished.callback(None) net_test_loader = NetTestLoader(('spam', 'ham')) net_test_loader.loadNetTestString(test_failing_twice) director = Director() director.netTestDone = net_test_done director.startNetTest(net_test_loader, None, no_yamloo=True) return finished
def test_run_test_fails_twice(self): finished = defer.Deferred() def net_test_done(net_test): summary_items = net_test.summary.items() self.assertEqual(len(summary_items), 50) for input_name, run_count in summary_items: self.assertEqual(run_count, 3) finished.callback(None) net_test_loader = NetTestLoader(('spam','ham')) net_test_loader.loadNetTestString(test_failing_twice) director = Director() director.netTestDone = net_test_done director.startNetTest(net_test_loader, None, no_yamloo=True) return finished
def test_net_test_inputs(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() nt = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) yield nt.initialize() for test_class, test_methods in nt.testCases: self.assertEqual(len(list(test_class.inputs)), 10)
def test_generate_measurements_size(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() net_test = NetTest(ntl.getTestCases(), ntl.getTestDetails(), None) yield net_test.initialize() measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)
def loadDeck(self, deckFile): with open(deckFile) as f: self.deckHash = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) for test in test_deck: try: nettest_path = nettest_to_path(test['options']['test_file']) except e.NetTestNotFound: log.err("Could not find %s" % test['options']['test_file']) log.msg("Skipping...") continue net_test_loader = NetTestLoader(test['options']['subargs'], test_file=nettest_path) if test['options']['collector']: net_test_loader.collector = test['options']['collector'] self.insert(net_test_loader)
def test_send_to_inputs_generator(self): """ Verify that a net test can send information back into an inputs generator. """ ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(generator_net_test) ntl.checkOptions() director = Director() self.filename = 'dummy_report.njson' d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): with open(self.filename) as report_file: all_report_entries = map(json.loads, report_file) results = [x['input'] for x in all_report_entries] self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0]) return d
def test_load_net_test_multiple(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(double_net_test_string) self.verifyMethods(ntl.testCases) self.verifyClasses(ntl.testCases, set(('DummyTestCaseA', 'DummyTestCaseB'))) ntl.checkOptions()
def runWithDirector(): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() log.start(global_options['logfile']) # contains (test_cases, options, cmd_line_options) test_list = [] if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: test_list.append( NetTestLoader(test['options']['subargs'], test_file=test['options']['test_file'])) else: log.debug("No test deck detected") test_list.append( NetTestLoader(global_options['subargs'], test_file=global_options['test_file'])) # check each test's usageOptions for net_test_loader in test_list: try: net_test_loader.checkOptions() except MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name) print net_test_loader.usageOptions().getUsage() sys.exit(2) except usage.UsageError, e: log.err(e) print net_test_loader.usageOptions().getUsage() sys.exit(2)
def test_generate_measurements_size(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() net_test = NetTest(ntl, None) net_test.initializeInputProcessor() measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)
def get_net_test_loader(test_options, test_file): """ Args: test_options: (dict) containing as keys the option names. test_file: (string) the path to the test_file to be run. Returns: an instance of :class:`ooni.nettest.NetTestLoader` with the specified test_file and the specified options. """ options = [] for k, v in test_options.items(): options.append('--'+k) options.append(v) net_test_loader = NetTestLoader(options, test_file=test_file) return net_test_loader
def _load_ooni(self, task_data): required_keys = ["test_name"] for required_key in required_keys: if required_key not in task_data: raise MissingTaskDataKey(required_key) # This raises e.NetTestNotFound, we let it go onto the caller nettest_path = nettest_to_path(task_data.pop("test_name"), self._arbitrary_paths) annotations = self._pop_option('annotations', task_data, {}) collector_address = self._pop_option('collector', task_data, None) try: self.output_path = self.global_options['reportfile'] except KeyError: self.output_path = task_data.pop('reportfile', None) if task_data.get('no-collector', False): collector_address = None elif config.reports.upload is False: collector_address = None net_test_loader = NetTestLoader( options_to_args(task_data), annotations=annotations, test_file=nettest_path ) if isinstance(collector_address, dict): net_test_loader.collector = CollectorClient( settings=collector_address ) elif collector_address is not None: net_test_loader.collector = CollectorClient( collector_address ) if (net_test_loader.collector is not None and net_test_loader.collector.backend_type == "onion"): self.requires_tor = True try: net_test_loader.checkOptions() if net_test_loader.requiresTor: self.requires_tor = True except e.MissingTestHelper: self.requires_bouncer = True self.ooni['net_test_loader'] = net_test_loader
def test_nettest_timeout(self): ntl = NetTestLoader(('-u', 'http://localhost:8007/')) ntl.loadNetTestString(http_net_test) ntl.checkOptions() director = Director() self.filename = 'dummy_report.yamloo' d = director.startNetTest(ntl, self.filename) @d.addCallback def complete(result): assert director.failedMeasurements == 1 return d
def test_net_test_completed_callback(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() director = Director() d = director.startNetTest('', ntl, [MockReporter()]) @d.addCallback def complete(result): print "IN here y0" self.assertEqual(result, None) self.assertEqual(director.successfulMeasurements, 20) return d
def test_net_test_completed_callback(self): ntl = NetTestLoader(dummyArgsWithFile) ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() director = Director() self.filename = 'dummy_report.yamloo' d = director.startNetTest(ntl, self.filename) @d.addCallback def complete(result): self.assertEqual(result, None) self.assertEqual(director.successfulMeasurements, 20) return d
def test_load_with_missing_required_option(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string_with_required_option) self.assertRaises(MissingRequiredOption, ntl.checkOptions)
def test_load_with_required_option(self): ntl = NetTestLoader(dummyArgsWithRequiredOptions) ntl.loadNetTestString(net_test_string_with_required_option) self.assertIsInstance(ntl, NetTestLoader)
def test_setup_local_options_in_test_cases(self): ntl = NetTestLoader(dummyArgs) ntl.loadNetTestString(net_test_string) ntl.checkOptions() self.assertEqual(dict(ntl.localOptions), dummyOptions)
def runWithDirector(logging=True, start_tor=True, check_incoherences=True): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() config.global_options = global_options config.set_paths() config.initialize_ooni_home() try: config.read_config_file(check_incoherences=check_incoherences) except errors.ConfigFileIncoherent: sys.exit(6) if global_options['verbose']: config.advanced.debug = True if not start_tor: config.advanced.start_tor = False if logging: log.start(global_options['logfile']) if config.privacy.includepcap: if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) else: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) director = Director() if global_options['list']: print "# Installed nettests" for net_test_id, net_test in director.getNetTests().items(): print "* %s (%s/%s)" % (net_test['name'], net_test['category'], net_test['id']) print " %s" % net_test['description'] sys.exit(0) elif global_options['printdeck']: del global_options['printdeck'] print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments" print yaml.safe_dump([{'options': global_options}]).strip() sys.exit(0) if global_options.get('annotations') is not None: annotations = {} for annotation in global_options["annotations"].split(","): pair = annotation.split(":") if len(pair) == 2: key = pair[0].strip() value = pair[1].strip() annotations[key] = value else: log.err("Invalid annotation: %s" % annotation) sys.exit(1) global_options["annotations"] = annotations if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None start_tor = False else: start_tor = True deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] if global_options['collector']: start_tor |= True try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file'], True) net_test_loader = NetTestLoader(global_options['subargs'], test_file=test_file) if global_options['collector']: net_test_loader.collector = global_options['collector'] deck.insert(net_test_loader) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() sys.exit(2) except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) sys.exit(3) except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) start_tor |= deck.requiresTor d = director.start(start_tor=start_tor, check_incoherences=check_incoherences) def setup_nettest(_): try: return deck.setup() except errors.UnableToLoadDeckInput as error: return defer.failure.Failure(error) def director_startup_handled_failures(failure): log.err("Could not start the director") failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, errors.CouldNotFindTestCollector, errors.ProbeIPUnknown, errors.InvalidInputFile, errors.ConfigFileIncoherent) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % global_options['collector']) log.msg( "Try with a different collector or disable collector reporting with -n" ) elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress): log.err("Invalid format for oonib collector address.") log.msg( "Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") elif isinstance(failure.value, errors.UnableToLoadDeckInput): log.err("Unable to fetch the required inputs for the test deck.") log.msg( "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues" ) elif isinstance(failure.value, errors.CouldNotFindTestHelper): log.err("Unable to obtain the required test helpers.") log.msg( "Try with a different bouncer or check that Tor is running properly." ) elif isinstance(failure.value, errors.CouldNotFindTestCollector): log.err("Could not find a valid collector.") log.msg( "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n." ) elif isinstance(failure.value, errors.ProbeIPUnknown): log.err("Failed to lookup probe IP address.") log.msg("Check your internet connection.") elif isinstance(failure.value, errors.InvalidInputFile): log.err("Invalid input file \"%s\"" % failure.value) elif isinstance(failure.value, errors.ConfigFileIncoherent): log.err("Incoherent config file") if config.advanced.debug: log.exception(failure) def director_startup_other_failures(failure): log.err("An unhandled exception occurred while starting the director!") log.exception(failure) # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): for net_test_loader in deck.netTestLoaders: # Decks can specify different collectors # for each net test, so that each NetTest # may be paired with a test_helper and its collector # However, a user can override this behavior by # specifying a collector from the command-line (-c). # If a collector is not specified in the deck, or the # deck is a singleton, the default collector set in # ooniprobe.conf will be used collector = None if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] elif 'collector' in config.reports \ and config.reports['collector']: collector = config.reports['collector'] elif net_test_loader.collector: collector = net_test_loader.collector if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning test_details = net_test_loader.testDetails test_details['annotations'] = global_options['annotations'] director.startNetTest(net_test_loader, global_options['reportfile'], collector) return director.allTestsDone def start(): d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_handled_failures) d.addErrback(director_startup_other_failures) return d return start()