def setupGlobalOptions(logging, start_tor, check_incoherences): global_options = parseOptions() config.global_options = global_options config.set_paths() config.initialize_ooni_home() try: config.read_config_file(check_incoherences=check_incoherences) except errors.ConfigFileIncoherent: sys.exit(6) if global_options['verbose']: config.advanced.debug = True if not start_tor: config.advanced.start_tor = False if logging: log.start(global_options['logfile']) if config.privacy.includepcap or global_options['pcapfile']: from ooni.utils.net import hasRawSocketPermission if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) else: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) global_options['check_incoherences'] = check_incoherences return global_options
def setupGlobalOptions(logging, start_tor, check_incoherences): global_options = parseOptions() config.global_options = global_options config.set_paths() config.initialize_ooni_home() try: config.read_config_file(check_incoherences=check_incoherences) except errors.ConfigFileIncoherent: sys.exit(6) if not config.is_initialized(): initializeOoniprobe(global_options) if global_options['verbose']: config.advanced.debug = True if not start_tor: config.advanced.start_tor = False if logging: log.start(global_options['logfile']) if config.privacy.includepcap or global_options['pcapfile']: from ooni.utils.net import hasRawSocketPermission if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) else: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) global_options['check_incoherences'] = check_incoherences return global_options
def askTraceroute(self): """ Perform a UDP traceroute to determine the probes IP address. """ if not hasRawSocketPermission(): raise errors.InsufficientPrivileges raise NotImplemented
def checkOptions(self): """ Call processTest and processOptions methods of each NetTestCase """ for klass in self.testClasses: options = self.usageOptions() try: options.parseOptions(self.options) except usage.UsageError: tb = sys.exc_info()[2] raise e.OONIUsageError(self), None, tb if options: klass.localOptions = options # XXX this class all needs to be refactored and this is kind of a # hack. self.setTestDetails() test_instance = klass() if test_instance.requiresRoot and not hasRawSocketPermission(): raise e.InsufficientPrivileges if test_instance.requiresTor: self.requiresTor = True test_instance.requirements() test_instance._checkRequiredOptions() test_instance._checkValidOptions()
def _checkTestClassOptions(self, test_class): if test_class.requiresRoot and not hasRawSocketPermission(): raise e.InsufficientPrivileges if test_class.requiresTor: self.requiresTor = True self._checkRequiredOptions(test_class) self._setTestHelpers(test_class) test_instance = netTestCaseFactory(test_class, self.localOptions)() test_instance.requirements()
def setUp(self): super(TestRunDirector, self).setUp() if not is_internet_connected(): self.skipTest("You must be connected to the internet to run this test") elif not hasRawSocketPermission(): self.skipTest("You must run this test as root or have the capabilities " "cap_net_admin,cap_net_raw+eip") config.tor.socks_port = 9050 config.tor.control_port = None self.filenames = ['example-input.txt'] with open('example-input.txt', 'w+') as f: f.write('http://torproject.org/\n') f.write('http://bridges.torproject.org/\n') f.write('http://blog.torproject.org/\n')
def setUp(self): super(TestRunDirector, self).setUp() if not is_internet_connected(): self.skipTest( "You must be connected to the internet to run this test") elif not hasRawSocketPermission(): self.skipTest( "You must run this test as root or have the capabilities " "cap_net_admin,cap_net_raw+eip") config.tor.socks_port = 9050 config.tor.control_port = None self.filenames = ['example-input.txt'] with open('example-input.txt', 'w+') as f: f.write('http://torproject.org/\n') f.write('http://bridges.torproject.org/\n') f.write('http://blog.torproject.org/\n')
def test_sniffing_activated(self): if not hasRawSocketPermission(): self.skipTest("You must run this test as root or have the " "capabilities cap_net_admin,cap_net_raw+eip") self.skipTest("Not properly set packet capture?") filename = os.path.abspath('test_report.pcap') self.filenames.append(filename) conf_file = os.path.abspath('fake_config.conf') with open(conf_file, 'w') as cfg: cfg.writelines(config_includepcap) self.filenames.append(conf_file) def verify_function(_): assert os.path.exists(filename) self.assertGreater(os.stat(filename).st_size, 0) yield self.run_helper('blocking/http_requests', ['-f', 'example-input.txt'], verify_function, ooni_args=['-f', conf_file]) config.scapyFactory.connectionLost('')
class BaseScapyTest(NetTestCase): """ The report of a test run with scapy looks like this: report: sent_packets: [ { 'raw_packet': BASE64Encoding of packet, 'summary': 'IP / TCP 192.168.2.66:ftp_data > 8.8.8.8:http S' } ] answered_packets: [] """ name = "Base Scapy Test" version = 0.1 requiresRoot = not hasRawSocketPermission() baseFlags = [ ['ipsrc', 's', 'Does *not* check if IP src and ICMP IP citation ' 'matches when processing answers'], ['seqack', 'k', 'Check if TCP sequence number and ACK match in the ' 'ICMP citation when processing answers'], ['ipid', 'i', 'Check if the IPID matches when processing answers']] def _setUp(self): super(BaseScapyTest, self)._setUp() if config.scapyFactory is None: log.debug("Scapy factory not set, registering it.") config.scapyFactory = ScapyFactory(config.advanced.interface) self.report['answer_flags'] = [] if self.localOptions['ipsrc']: config.checkIPsrc = 0 else: self.report['answer_flags'].append('ipsrc') config.checkIPsrc = 1 if self.localOptions['ipid']: self.report['answer_flags'].append('ipid') config.checkIPID = 1 else: config.checkIPID = 0 # XXX we don't support strict matching # since (from scapy's documentation), some stacks have a bug for which # the bytes in the IPID are swapped. # Perhaps in the future we will want to have more fine grained control # over this. if self.localOptions['seqack']: self.report['answer_flags'].append('seqack') config.check_TCPerror_seqack = 1 else: config.check_TCPerror_seqack = 0 self.report['sent_packets'] = [] self.report['answered_packets'] = [] def finishedSendReceive(self, packets): """ This gets called when all packets have been sent and received. """ answered, unanswered = packets for snd, rcv in answered: log.debug("Writing report for scapy test") sent_packet = snd received_packet = rcv if not config.privacy.includeip: log.debug("Detected you would not like to " "include your ip in the report") log.debug( "Stripping source and destination IPs from the reports") sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' self.report['sent_packets'].append(representPacket(sent_packet)) self.report['answered_packets'].append(representPacket(received_packet)) return packets def sr(self, packets, timeout=None, *arg, **kw): """ Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ scapySender = ScapySender(timeout=timeout) config.scapyFactory.registerProtocol(scapySender) log.debug("Using sending with hash %s" % scapySender.__hash__) d = scapySender.startSending(packets) d.addCallback(self.finishedSendReceive) return d def sr1(self, packets, *arg, **kw): def done(packets): """ We do this so that the returned value is only the one packet that we expected a response for, identical to the scapy implementation of sr1. """ try: return packets[0][0][1] except IndexError: log.err("Got no response...") return packets scapySender = ScapySender() scapySender.expected_answers = 1 config.scapyFactory.registerProtocol(scapySender) log.debug("Running sr1") d = scapySender.startSending(packets) log.debug("Started to send") d.addCallback(self.finishedSendReceive) d.addCallback(done) return d def send(self, packets, *arg, **kw): """ Wrapper around scapy.sendrecv.send for sending of packets at layer 3 """ scapySender = ScapySender() config.scapyFactory.registerProtocol(scapySender) scapySender.startSending(packets) scapySender.stopSending() for sent_packet in packets: self.report['sent_packets'].append(representPacket(sent_packet))
def runWithDirector(logging=True, start_tor=True, check_incoherences=True): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() config.global_options = global_options config.set_paths() config.initialize_ooni_home() try: config.read_config_file(check_incoherences=check_incoherences) except errors.ConfigFileIncoherent: sys.exit(6) if global_options['verbose']: config.advanced.debug = True if not start_tor: config.advanced.start_tor = False if logging: log.start(global_options['logfile']) if config.privacy.includepcap: if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) else: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) director = Director() if global_options['list']: print "# Installed nettests" for net_test_id, net_test in director.getNetTests().items(): print "* %s (%s/%s)" % (net_test['name'], net_test['category'], net_test['id']) print " %s" % net_test['description'] sys.exit(0) elif global_options['printdeck']: del global_options['printdeck'] print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments" print yaml.safe_dump([{'options': global_options}]).strip() sys.exit(0) if global_options.get('annotations') is not None: annotations = {} for annotation in global_options["annotations"].split(","): pair = annotation.split(":") if len(pair) == 2: key = pair[0].strip() value = pair[1].strip() annotations[key] = value else: log.err("Invalid annotation: %s" % annotation) sys.exit(1) global_options["annotations"] = annotations if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None start_tor = False else: start_tor = True deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] if global_options['collector']: start_tor |= True try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file'], True) net_test_loader = NetTestLoader(global_options['subargs'], test_file=test_file) if global_options['collector']: net_test_loader.collector = global_options['collector'] deck.insert(net_test_loader) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() sys.exit(2) except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) sys.exit(3) except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) start_tor |= deck.requiresTor d = director.start(start_tor=start_tor, check_incoherences=check_incoherences) def setup_nettest(_): try: return deck.setup() except errors.UnableToLoadDeckInput as error: return defer.failure.Failure(error) def director_startup_handled_failures(failure): log.err("Could not start the director") failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, errors.CouldNotFindTestCollector, errors.ProbeIPUnknown, errors.InvalidInputFile, errors.ConfigFileIncoherent) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % global_options['collector']) log.msg( "Try with a different collector or disable collector reporting with -n" ) elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress): log.err("Invalid format for oonib collector address.") log.msg( "Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") elif isinstance(failure.value, errors.UnableToLoadDeckInput): log.err("Unable to fetch the required inputs for the test deck.") log.msg( "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues" ) elif isinstance(failure.value, errors.CouldNotFindTestHelper): log.err("Unable to obtain the required test helpers.") log.msg( "Try with a different bouncer or check that Tor is running properly." ) elif isinstance(failure.value, errors.CouldNotFindTestCollector): log.err("Could not find a valid collector.") log.msg( "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n." ) elif isinstance(failure.value, errors.ProbeIPUnknown): log.err("Failed to lookup probe IP address.") log.msg("Check your internet connection.") elif isinstance(failure.value, errors.InvalidInputFile): log.err("Invalid input file \"%s\"" % failure.value) elif isinstance(failure.value, errors.ConfigFileIncoherent): log.err("Incoherent config file") if config.advanced.debug: log.exception(failure) def director_startup_other_failures(failure): log.err("An unhandled exception occurred while starting the director!") log.exception(failure) # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): for net_test_loader in deck.netTestLoaders: # Decks can specify different collectors # for each net test, so that each NetTest # may be paired with a test_helper and its collector # However, a user can override this behavior by # specifying a collector from the command-line (-c). # If a collector is not specified in the deck, or the # deck is a singleton, the default collector set in # ooniprobe.conf will be used collector = None if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] elif 'collector' in config.reports \ and config.reports['collector']: collector = config.reports['collector'] elif net_test_loader.collector: collector = net_test_loader.collector if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning test_details = net_test_loader.testDetails test_details['annotations'] = global_options['annotations'] director.startNetTest(net_test_loader, global_options['reportfile'], collector) return director.allTestsDone def start(): d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_handled_failures) d.addErrback(director_startup_other_failures) return d return start()
def runWithDirector(logging=True, start_tor=True, check_incoherences=True): """ Instance the director, parse command line options and start an ooniprobe test! """ global_options = parseOptions() config.global_options = global_options config.set_paths() config.initialize_ooni_home() try: config.read_config_file(check_incoherences=check_incoherences) except errors.ConfigFileIncoherent: sys.exit(6) if global_options['verbose']: config.advanced.debug = True if not start_tor: config.advanced.start_tor = False if logging: log.start(global_options['logfile']) if config.privacy.includepcap: if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) else: log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) director = Director() if global_options['list']: print "# Installed nettests" for net_test_id, net_test in director.getNetTests().items(): print "* %s (%s/%s)" % (net_test['name'], net_test['category'], net_test['id']) print " %s" % net_test['description'] sys.exit(0) elif global_options['printdeck']: del global_options['printdeck'] print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments" print yaml.safe_dump([{'options': global_options}]).strip() sys.exit(0) if global_options.get('annotations') is not None: annotations = {} for annotation in global_options["annotations"].split(","): pair = annotation.split(":") if len(pair) == 2: key = pair[0].strip() value = pair[1].strip() annotations[key] = value else: log.err("Invalid annotation: %s" % annotation) sys.exit(1) global_options["annotations"] = annotations if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None start_tor = False else: start_tor = True deck = Deck(no_collector=global_options['no-collector']) deck.bouncer = global_options['bouncer'] if global_options['collector']: start_tor |= True try: if global_options['testdeck']: deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file'], True) net_test_loader = NetTestLoader(global_options['subargs'], test_file=test_file) if global_options['collector']: net_test_loader.collector = global_options['collector'] deck.insert(net_test_loader) except errors.MissingRequiredOption as option_name: log.err('Missing required option: "%s"' % option_name) incomplete_net_test_loader = option_name.net_test_loader print incomplete_net_test_loader.usageOptions().getUsage() sys.exit(2) except errors.NetTestNotFound as path: log.err('Requested NetTest file not found (%s)' % path) sys.exit(3) except errors.OONIUsageError as e: log.err(e) print e.net_test_loader.usageOptions().getUsage() sys.exit(4) except Exception as e: if config.advanced.debug: log.exception(e) log.err(e) sys.exit(5) start_tor |= deck.requiresTor d = director.start(start_tor=start_tor, check_incoherences=check_incoherences) def setup_nettest(_): try: return deck.setup() except errors.UnableToLoadDeckInput as error: return defer.failure.Failure(error) def director_startup_handled_failures(failure): log.err("Could not start the director") failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, errors.CouldNotFindTestCollector, errors.ProbeIPUnknown, errors.InvalidInputFile, errors.ConfigFileIncoherent) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % global_options['collector']) log.msg( "Try with a different collector or disable collector reporting with -n") elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress): log.err("Invalid format for oonib collector address.") log.msg( "Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") elif isinstance(failure.value, errors.UnableToLoadDeckInput): log.err("Unable to fetch the required inputs for the test deck.") log.msg( "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues") elif isinstance(failure.value, errors.CouldNotFindTestHelper): log.err("Unable to obtain the required test helpers.") log.msg( "Try with a different bouncer or check that Tor is running properly.") elif isinstance(failure.value, errors.CouldNotFindTestCollector): log.err("Could not find a valid collector.") log.msg( "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.") elif isinstance(failure.value, errors.ProbeIPUnknown): log.err("Failed to lookup probe IP address.") log.msg("Check your internet connection.") elif isinstance(failure.value, errors.InvalidInputFile): log.err("Invalid input file \"%s\"" % failure.value) elif isinstance(failure.value, errors.ConfigFileIncoherent): log.err("Incoherent config file") if config.advanced.debug: log.exception(failure) def director_startup_other_failures(failure): log.err("An unhandled exception occurred while starting the director!") log.exception(failure) # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): for net_test_loader in deck.netTestLoaders: # Decks can specify different collectors # for each net test, so that each NetTest # may be paired with a test_helper and its collector # However, a user can override this behavior by # specifying a collector from the command-line (-c). # If a collector is not specified in the deck, or the # deck is a singleton, the default collector set in # ooniprobe.conf will be used collector = None if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] elif 'collector' in config.reports \ and config.reports['collector']: collector = config.reports['collector'] elif net_test_loader.collector: collector = net_test_loader.collector if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning test_details = net_test_loader.testDetails test_details['annotations'] = global_options['annotations'] director.startNetTest(net_test_loader, global_options['reportfile'], collector) return director.allTestsDone def start(): d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_handled_failures) d.addErrback(director_startup_other_failures) return d return start()