def setupCollector(global_options, collector_client): from ooni.backend_client import CollectorClient if global_options['collector']: collector_client = CollectorClient(global_options['collector']) elif config.reports.get('collector', None) is not None: collector_client = CollectorClient(config.reports['collector']) if not collector_client.isSupported(): raise errors.CollectorUnsupported return collector_client
def _load_ooni(self, task_data): required_keys = ["test_name"] for required_key in required_keys: if required_key not in task_data: raise MissingTaskDataKey(required_key) # This raises e.NetTestNotFound, we let it go onto the caller nettest_path = nettest_to_path(task_data.pop("test_name"), self._arbitrary_paths) annotations = self._pop_option('annotations', task_data, {}) collector_address = self._pop_option('collector', task_data, None) try: self.output_path = self.global_options['reportfile'] except KeyError: self.output_path = task_data.pop('reportfile', None) if task_data.get('no-collector', False): collector_address = None elif config.reports.upload is False: collector_address = None net_test_loader = NetTestLoader( options_to_args(task_data), annotations=annotations, test_file=nettest_path ) if isinstance(collector_address, dict): net_test_loader.collector = CollectorClient( settings=collector_address ) elif collector_address is not None: net_test_loader.collector = CollectorClient( collector_address ) if (net_test_loader.collector is not None and net_test_loader.collector.backend_type == "onion"): self.requires_tor = True try: net_test_loader.checkOptions() if net_test_loader.requiresTor: self.requires_tor = True except e.MissingTestHelper: self.requires_bouncer = True self.ooni['net_test_loader'] = net_test_loader
def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() collector_client = None if collector: collector_client = CollectorClient(address=collector) log.msg("Attempting to upload %s" % report_file) with open(config.report_log_file) as f: report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) if bouncer and collector_client is None: collector_client = yield lookup_collector_client( report.header, bouncer) if collector_client is None: try: collector_settings = report_log[report_file]['collector'] if collector_settings is None: log.msg("Skipping uploading of %s since this measurement " "was run by specifying no collector." % report_file) defer.returnValue(None) elif isinstance(collector_settings, dict): collector_client = CollectorClient(settings=collector_settings) elif isinstance(collector_settings, str): collector_client = CollectorClient(address=collector_settings) except KeyError: log.msg("Could not find %s in reporting.yaml. Looking up " "collector with canonical bouncer." % report_file) collector_client = yield lookup_collector_client( report.header, CANONICAL_BOUNCER_ONION) oonib_reporter = OONIBReporter(report.header, collector_client) log.msg("Creating report for %s with %s" % (report_file, collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector_client.settings, report_id) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) sys.stdout.write('.') sys.stdout.flush() log.msg("Closing report") yield oonib_reporter.finish() yield oonib_report_log.closed(report_file)
def loadDeck(self, deckFile, global_options={}): with open(deckFile) as f: self.id = sha256(f.read()).hexdigest() f.seek(0) test_deck = yaml.safe_load(f) for test in test_deck: try: nettest_path = nettest_to_path(test['options']['test_file']) except e.NetTestNotFound: log.err("Could not find %s" % test['options']['test_file']) log.msg("Skipping...") continue annotations = test['options'].get('annotations', {}) if global_options.get('annotations') is not None: annotations = global_options["annotations"] collector_address = test['options'].get('collector', None) if global_options.get('collector') is not None: collector_address = global_options['collector'] net_test_loader = NetTestLoader(test['options']['subargs'], annotations=annotations, test_file=nettest_path) if collector_address is not None: net_test_loader.collector = CollectorClient(collector_address) if test['options'].get('bouncer', None) is not None: self.bouncer = self._BouncerClient(test['options']['bouncer']) if self.bouncer.backend_type == "onion": self.requiresTor = True self.insert(net_test_loader)
def get_reachable_collector(collector_address, collector_alternate, preferred_backend): for collector_settings in sort_addresses_by_priority( collector_address, collector_alternate, preferred_backend): collector = CollectorClient(settings=collector_settings) if not collector.isSupported(): log.err( "Unsupported %s collector %s" % (collector_settings['type'], collector_settings['address'])) continue reachable = yield collector.isReachable() if not reachable: log.err( "Unreachable %s collector %s" % (collector_settings['type'], collector_settings['address'])) continue defer.returnValue(collector) raise e.NoReachableCollectors
def test_backend_client_validates_url(self): raised = False try: cc = CollectorClient(settings={ "type": "onion", "address": "http://invalid.onion" }) except Exception as exc: raised = True self.assertIsInstance(exc, e.InvalidAddress) self.assertTrue(raised)
def lookup_collector_client(report_header, bouncer): oonib_client = BouncerClient(bouncer) net_tests = [{ 'test-helpers': [], 'input-hashes': report_header['input_hashes'], 'name': report_header['test_name'], 'version': report_header['test_version'], }] result = yield oonib_client.lookupTestCollector(net_tests) collector_client = CollectorClient( address=result['net-tests'][0]['collector']) defer.returnValue(collector_client)
def setUp(self): super(TestEnd2EndBackendClient, self).setUp() host = '127.0.0.1' port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) s.shutdown(2) data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir if os.path.exists(data_dir): shutil.rmtree(data_dir) os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) except Exception: self.skipTest( "OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)" ) self.collector_client = CollectorClient('http://' + host + ':' + str(port))
def setUp(self): super(TestEnd2EndBackendClient, self).setUp() host = '127.0.0.1' port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) s.shutdown(2) data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir if os.path.exists(data_dir): shutil.rmtree(data_dir) os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) except Exception: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.collector_client = CollectorClient('http://' + host + ':' + str(port))
class TestEnd2EndBackendClient(ConfigTestCase): def setUp(self): super(TestEnd2EndBackendClient, self).setUp() host = '127.0.0.1' port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) s.shutdown(2) data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir if os.path.exists(data_dir): shutil.rmtree(data_dir) os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) except Exception: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.collector_client = CollectorClient('http://' + host + ':' + str(port)) @defer.inlineCallbacks def test_query(self): res = yield self.collector_client.queryBackend('GET', '/policy/input') self.assertTrue(isinstance(res, list)) @defer.inlineCallbacks def test_get_input_list(self): input_list = yield self.collector_client.getInputList() self.assertTrue(isinstance(input_list, list)) @defer.inlineCallbacks def test_get_input_descriptor(self): input_descriptor = yield self.collector_client.getInput(input_id) for key in ['name', 'description', 'version', 'author', 'date', 'id']: self.assertTrue(hasattr(input_descriptor, key)) @defer.inlineCallbacks def test_download_input(self): yield self.collector_client.downloadInput(input_id) @defer.inlineCallbacks def test_get_deck_list(self): deck_list = yield self.collector_client.getDeckList() self.assertTrue(isinstance(deck_list, list)) @defer.inlineCallbacks def test_get_deck_descriptor(self): deck_descriptor = yield self.collector_client.getDeck(deck_id) for key in ['name', 'description', 'version', 'author', 'date', 'id']: self.assertTrue(hasattr(deck_descriptor, key)) @defer.inlineCallbacks def test_download_deck(self): yield self.collector_client.downloadDeck(deck_id) def test_lookup_invalid_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') return self.failUnlessFailure( bouncer_client.lookupTestHelpers([ 'sdadsadsa', 'dns' ]), e.CouldNotFindTestHelper) @defer.inlineCallbacks def test_lookup_no_test_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') required_helpers = [] helpers = yield bouncer_client.lookupTestHelpers(required_helpers) self.assertTrue('default' in helpers.keys()) @defer.inlineCallbacks def test_lookup_test_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') required_helpers = [u'http-return-json-headers', u'dns'] helpers = yield bouncer_client.lookupTestHelpers(required_helpers) self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http')) self.assertTrue(int(helpers['dns']['address'].split('.')[0])) @defer.inlineCallbacks def test_input_descriptor_not_found(self): yield self.assertFailure(self.collector_client.queryBackend('GET', '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound) @defer.inlineCallbacks def test_http_errors(self): yield self.assertFailure(self.collector_client.queryBackend('PUT', '/policy/input'), error.Error) @defer.inlineCallbacks def test_create_report(self): res = yield self.collector_client.queryBackend('POST', '/report', { 'software_name': 'spam', 'software_version': '2.0', 'probe_asn': 'AS0', 'probe_cc': 'ZZ', 'test_name': 'foobar', 'test_version': '1.0', 'input_hashes': [] }) assert isinstance(res['report_id'], unicode) @defer.inlineCallbacks def test_report_lifecycle(self): res = yield self.collector_client.queryBackend('POST', '/report', { 'software_name': 'spam', 'software_version': '2.0', 'probe_asn': 'AS0', 'probe_cc': 'ZZ', 'test_name': 'foobar', 'test_version': '1.0', 'input_hashes': [] }) report_id = str(res['report_id']) res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { 'content': '---\nspam: ham\n...\n' }) res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { 'content': '---\nspam: ham\n...\n' }) res = yield self.collector_client.queryBackend('POST', '/report/' + report_id + '/close')
class TestEnd2EndBackendClient(ConfigTestCase): def setUp(self): super(TestEnd2EndBackendClient, self).setUp() host = '127.0.0.1' port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) s.shutdown(2) data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir if os.path.exists(data_dir): shutil.rmtree(data_dir) os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) except Exception: self.skipTest( "OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)" ) self.collector_client = CollectorClient('http://' + host + ':' + str(port)) @defer.inlineCallbacks def test_query(self): res = yield self.collector_client.queryBackend('GET', '/policy/input') self.assertTrue(isinstance(res, list)) @defer.inlineCallbacks def test_get_input_list(self): input_list = yield self.collector_client.getInputList() self.assertTrue(isinstance(input_list, list)) @defer.inlineCallbacks def test_get_input_descriptor(self): input_descriptor = yield self.collector_client.getInput(input_id) for key in ['name', 'description', 'version', 'author', 'date', 'id']: self.assertTrue(hasattr(input_descriptor, key)) @defer.inlineCallbacks def test_download_input(self): yield self.collector_client.downloadInput(input_id) def test_lookup_invalid_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') return self.failUnlessFailure( bouncer_client.lookupTestHelpers(['sdadsadsa', 'dns']), e.CouldNotFindTestHelper) @defer.inlineCallbacks def test_lookup_no_test_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') required_helpers = [] helpers = yield bouncer_client.lookupTestHelpers(required_helpers) self.assertTrue('default' in helpers.keys()) @defer.inlineCallbacks def test_lookup_test_helpers(self): bouncer_client = BouncerClient('http://127.0.0.1:8888') required_helpers = [u'http-return-json-headers', u'dns'] helpers = yield bouncer_client.lookupTestHelpers(required_helpers) self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) self.assertTrue( helpers['http-return-json-headers']['address'].startswith('http')) self.assertTrue(int(helpers['dns']['address'].split('.')[0])) @defer.inlineCallbacks def test_input_descriptor_not_found(self): yield self.assertFailure( self.collector_client.queryBackend('GET', '/input/' + 'a' * 64), e.OONIBInputDescriptorNotFound) @defer.inlineCallbacks def test_http_errors(self): yield self.assertFailure( self.collector_client.queryBackend('PUT', '/policy/input'), error.Error) @defer.inlineCallbacks def test_create_report(self): res = yield self.collector_client.queryBackend( 'POST', '/report', { 'software_name': 'spam', 'software_version': '2.0', 'probe_asn': 'AS0', 'probe_cc': 'ZZ', 'test_name': 'foobar', 'test_version': '1.0', 'input_hashes': [] }) assert isinstance(res['report_id'], unicode) @defer.inlineCallbacks def test_report_lifecycle(self): res = yield self.collector_client.queryBackend( 'POST', '/report', { 'software_name': 'spam', 'software_version': '2.0', 'probe_asn': 'AS0', 'probe_cc': 'ZZ', 'test_name': 'foobar', 'test_version': '1.0', 'input_hashes': [] }) report_id = str(res['report_id']) res = yield self.collector_client.queryBackend( 'POST', '/report/' + report_id, {'content': '---\nspam: ham\n...\n'}) res = yield self.collector_client.queryBackend( 'POST', '/report/' + report_id, {'content': '---\nspam: ham\n...\n'}) res = yield self.collector_client.queryBackend( 'POST', '/report/' + report_id + '/close')
def upload(report_file, collector=None, bouncer=None, measurement_id=None): oonib_report_log = OONIBReportLog() collector_client = None if collector: collector_client = CollectorClient(address=collector) try: # Try to guess the measurement_id from the file path measurement_id = report_path_to_id(report_file) except NoIDFound: pass log.msg("Attempting to upload %s" % report_file) if report_file.endswith(".njson"): report = NJSONReportLoader(report_file) else: log.warn("Uploading of YAML formatted reports will be dropped in " "future versions") report = YAMLReportLoader(report_file) if bouncer and collector_client is None: collector_client = yield lookup_collector_client( report.header, bouncer) if collector_client is None: if measurement_id: report_log = yield oonib_report_log.get_report_log(measurement_id) collector_settings = report_log['collector'] print(collector_settings) if collector_settings is None or len(collector_settings) == 0: log.warn("Skipping uploading of %s since this measurement " "was run by specifying no collector." % report_file) defer.returnValue(None) elif isinstance(collector_settings, dict): collector_client = CollectorClient(settings=collector_settings) elif isinstance(collector_settings, str): collector_client = CollectorClient(address=collector_settings) else: log.msg("Looking up collector with canonical bouncer." % report_file) collector_client = yield lookup_collector_client( report.header, CANONICAL_BOUNCER_ONION) oonib_reporter = OONIBReporter(report.header, collector_client) log.msg("Creating report for %s with %s" % (report_file, collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id if measurement_id: log.debug("Marking it as created") yield oonib_report_log.created(measurement_id, collector_client.settings) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) log.msg("Written entry") log.msg("Closing report") yield oonib_reporter.finish() if measurement_id: log.debug("Closing log") yield oonib_report_log.closed(measurement_id)