def setupCollector(global_options, collector_client):
from ooni.backend_client import CollectorClient
if global_options['collector']:
collector_client = CollectorClient(global_options['collector'])
elif config.reports.get('collector', None) is not None:
collector_client = CollectorClient(config.reports['collector'])
if not collector_client.isSupported():
raise errors.CollectorUnsupported
return collector_client
def setupCollector(global_options, collector_client):
from ooni.backend_client import CollectorClient
if global_options['collector']:
collector_client = CollectorClient(global_options['collector'])
elif config.reports.get('collector', None) is not None:
collector_client = CollectorClient(config.reports['collector'])
if not collector_client.isSupported():
raise errors.CollectorUnsupported
return collector_client
def _load_ooni(self, task_data):
required_keys = ["test_name"]
for required_key in required_keys:
if required_key not in task_data:
raise MissingTaskDataKey(required_key)
# This raises e.NetTestNotFound, we let it go onto the caller
nettest_path = nettest_to_path(task_data.pop("test_name"),
self._arbitrary_paths)
annotations = self._pop_option('annotations', task_data, {})
collector_address = self._pop_option('collector', task_data, None)
try:
self.output_path = self.global_options['reportfile']
except KeyError:
self.output_path = task_data.pop('reportfile', None)
if task_data.get('no-collector', False):
collector_address = None
elif config.reports.upload is False:
collector_address = None
net_test_loader = NetTestLoader(
options_to_args(task_data),
annotations=annotations,
test_file=nettest_path
)
if isinstance(collector_address, dict):
net_test_loader.collector = CollectorClient(
settings=collector_address
)
elif collector_address is not None:
net_test_loader.collector = CollectorClient(
collector_address
)
if (net_test_loader.collector is not None and
net_test_loader.collector.backend_type == "onion"):
self.requires_tor = True
try:
net_test_loader.checkOptions()
if net_test_loader.requiresTor:
self.requires_tor = True
except e.MissingTestHelper:
self.requires_bouncer = True
self.ooni['net_test_loader'] = net_test_loader
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
try:
collector_settings = report_log[report_file]['collector']
if collector_settings is None:
log.msg("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
except KeyError:
log.msg("Could not find %s in reporting.yaml. Looking up "
"collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector_client.settings,
report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def loadDeck(self, deckFile, global_options={}):
with open(deckFile) as f:
self.id = sha256(f.read()).hexdigest()
f.seek(0)
test_deck = yaml.safe_load(f)
for test in test_deck:
try:
nettest_path = nettest_to_path(test['options']['test_file'])
except e.NetTestNotFound:
log.err("Could not find %s" % test['options']['test_file'])
log.msg("Skipping...")
continue
annotations = test['options'].get('annotations', {})
if global_options.get('annotations') is not None:
annotations = global_options["annotations"]
collector_address = test['options'].get('collector', None)
if global_options.get('collector') is not None:
collector_address = global_options['collector']
net_test_loader = NetTestLoader(test['options']['subargs'],
annotations=annotations,
test_file=nettest_path)
if collector_address is not None:
net_test_loader.collector = CollectorClient(collector_address)
if test['options'].get('bouncer', None) is not None:
self.bouncer = self._BouncerClient(test['options']['bouncer'])
if self.bouncer.backend_type == "onion":
self.requiresTor = True
self.insert(net_test_loader)
def get_reachable_collector(collector_address, collector_alternate,
preferred_backend):
for collector_settings in sort_addresses_by_priority(
collector_address, collector_alternate, preferred_backend):
collector = CollectorClient(settings=collector_settings)
if not collector.isSupported():
log.err(
"Unsupported %s collector %s" %
(collector_settings['type'], collector_settings['address']))
continue
reachable = yield collector.isReachable()
if not reachable:
log.err(
"Unreachable %s collector %s" %
(collector_settings['type'], collector_settings['address']))
continue
defer.returnValue(collector)
raise e.NoReachableCollectors
def test_backend_client_validates_url(self):
raised = False
try:
cc = CollectorClient(settings={
"type": "onion",
"address": "http://invalid.onion"
})
except Exception as exc:
raised = True
self.assertIsInstance(exc, e.InvalidAddress)
self.assertTrue(raised)
def lookup_collector_client(report_header, bouncer):
oonib_client = BouncerClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report_header['input_hashes'],
'name': report_header['test_name'],
'version': report_header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(net_tests)
collector_client = CollectorClient(
address=result['net-tests'][0]['collector'])
defer.returnValue(collector_client)
def setUp(self):
super(TestEnd2EndBackendClient, self).setUp()
host = '127.0.0.1'
port = 8889
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(2)
data_dir = '/tmp/testooni'
config.advanced.data_dir = data_dir
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
os.mkdir(data_dir)
os.mkdir(os.path.join(data_dir, 'inputs'))
os.mkdir(os.path.join(data_dir, 'decks'))
except Exception:
self.skipTest(
"OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)"
)
self.collector_client = CollectorClient('http://' + host + ':' +
str(port))
def setUp(self):
super(TestEnd2EndBackendClient, self).setUp()
host = '127.0.0.1'
port = 8889
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(2)
data_dir = '/tmp/testooni'
config.advanced.data_dir = data_dir
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
os.mkdir(data_dir)
os.mkdir(os.path.join(data_dir, 'inputs'))
os.mkdir(os.path.join(data_dir, 'decks'))
except Exception:
self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
self.collector_client = CollectorClient('http://' + host + ':' + str(port))
class TestEnd2EndBackendClient(ConfigTestCase):
def setUp(self):
super(TestEnd2EndBackendClient, self).setUp()
host = '127.0.0.1'
port = 8889
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(2)
data_dir = '/tmp/testooni'
config.advanced.data_dir = data_dir
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
os.mkdir(data_dir)
os.mkdir(os.path.join(data_dir, 'inputs'))
os.mkdir(os.path.join(data_dir, 'decks'))
except Exception:
self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
self.collector_client = CollectorClient('http://' + host + ':' + str(port))
@defer.inlineCallbacks
def test_query(self):
res = yield self.collector_client.queryBackend('GET', '/policy/input')
self.assertTrue(isinstance(res, list))
@defer.inlineCallbacks
def test_get_input_list(self):
input_list = yield self.collector_client.getInputList()
self.assertTrue(isinstance(input_list, list))
@defer.inlineCallbacks
def test_get_input_descriptor(self):
input_descriptor = yield self.collector_client.getInput(input_id)
for key in ['name', 'description',
'version', 'author', 'date', 'id']:
self.assertTrue(hasattr(input_descriptor, key))
@defer.inlineCallbacks
def test_download_input(self):
yield self.collector_client.downloadInput(input_id)
@defer.inlineCallbacks
def test_get_deck_list(self):
deck_list = yield self.collector_client.getDeckList()
self.assertTrue(isinstance(deck_list, list))
@defer.inlineCallbacks
def test_get_deck_descriptor(self):
deck_descriptor = yield self.collector_client.getDeck(deck_id)
for key in ['name', 'description',
'version', 'author', 'date', 'id']:
self.assertTrue(hasattr(deck_descriptor, key))
@defer.inlineCallbacks
def test_download_deck(self):
yield self.collector_client.downloadDeck(deck_id)
def test_lookup_invalid_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
return self.failUnlessFailure(
bouncer_client.lookupTestHelpers([
'sdadsadsa', 'dns'
]), e.CouldNotFindTestHelper)
@defer.inlineCallbacks
def test_lookup_no_test_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
required_helpers = []
helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
self.assertTrue('default' in helpers.keys())
@defer.inlineCallbacks
def test_lookup_test_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
required_helpers = [u'http-return-json-headers', u'dns']
helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))
self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http'))
self.assertTrue(int(helpers['dns']['address'].split('.')[0]))
@defer.inlineCallbacks
def test_input_descriptor_not_found(self):
yield self.assertFailure(self.collector_client.queryBackend('GET',
'/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)
@defer.inlineCallbacks
def test_http_errors(self):
yield self.assertFailure(self.collector_client.queryBackend('PUT',
'/policy/input'), error.Error)
@defer.inlineCallbacks
def test_create_report(self):
res = yield self.collector_client.queryBackend('POST', '/report', {
'software_name': 'spam',
'software_version': '2.0',
'probe_asn': 'AS0',
'probe_cc': 'ZZ',
'test_name': 'foobar',
'test_version': '1.0',
'input_hashes': []
})
assert isinstance(res['report_id'], unicode)
@defer.inlineCallbacks
def test_report_lifecycle(self):
res = yield self.collector_client.queryBackend('POST', '/report', {
'software_name': 'spam',
'software_version': '2.0',
'probe_asn': 'AS0',
'probe_cc': 'ZZ',
'test_name': 'foobar',
'test_version': '1.0',
'input_hashes': []
})
report_id = str(res['report_id'])
res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, {
'content': '---\nspam: ham\n...\n'
})
res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, {
'content': '---\nspam: ham\n...\n'
})
res = yield self.collector_client.queryBackend('POST', '/report/' + report_id +
'/close')
class TestEnd2EndBackendClient(ConfigTestCase):
def setUp(self):
super(TestEnd2EndBackendClient, self).setUp()
host = '127.0.0.1'
port = 8889
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(2)
data_dir = '/tmp/testooni'
config.advanced.data_dir = data_dir
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
os.mkdir(data_dir)
os.mkdir(os.path.join(data_dir, 'inputs'))
os.mkdir(os.path.join(data_dir, 'decks'))
except Exception:
self.skipTest(
"OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)"
)
self.collector_client = CollectorClient('http://' + host + ':' +
str(port))
@defer.inlineCallbacks
def test_query(self):
res = yield self.collector_client.queryBackend('GET', '/policy/input')
self.assertTrue(isinstance(res, list))
@defer.inlineCallbacks
def test_get_input_list(self):
input_list = yield self.collector_client.getInputList()
self.assertTrue(isinstance(input_list, list))
@defer.inlineCallbacks
def test_get_input_descriptor(self):
input_descriptor = yield self.collector_client.getInput(input_id)
for key in ['name', 'description', 'version', 'author', 'date', 'id']:
self.assertTrue(hasattr(input_descriptor, key))
@defer.inlineCallbacks
def test_download_input(self):
yield self.collector_client.downloadInput(input_id)
def test_lookup_invalid_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
return self.failUnlessFailure(
bouncer_client.lookupTestHelpers(['sdadsadsa', 'dns']),
e.CouldNotFindTestHelper)
@defer.inlineCallbacks
def test_lookup_no_test_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
required_helpers = []
helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
self.assertTrue('default' in helpers.keys())
@defer.inlineCallbacks
def test_lookup_test_helpers(self):
bouncer_client = BouncerClient('http://127.0.0.1:8888')
required_helpers = [u'http-return-json-headers', u'dns']
helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
self.assertEqual(set(helpers.keys()),
set(required_helpers + [u'default']))
self.assertTrue(
helpers['http-return-json-headers']['address'].startswith('http'))
self.assertTrue(int(helpers['dns']['address'].split('.')[0]))
@defer.inlineCallbacks
def test_input_descriptor_not_found(self):
yield self.assertFailure(
self.collector_client.queryBackend('GET', '/input/' + 'a' * 64),
e.OONIBInputDescriptorNotFound)
@defer.inlineCallbacks
def test_http_errors(self):
yield self.assertFailure(
self.collector_client.queryBackend('PUT', '/policy/input'),
error.Error)
@defer.inlineCallbacks
def test_create_report(self):
res = yield self.collector_client.queryBackend(
'POST', '/report', {
'software_name': 'spam',
'software_version': '2.0',
'probe_asn': 'AS0',
'probe_cc': 'ZZ',
'test_name': 'foobar',
'test_version': '1.0',
'input_hashes': []
})
assert isinstance(res['report_id'], unicode)
@defer.inlineCallbacks
def test_report_lifecycle(self):
res = yield self.collector_client.queryBackend(
'POST', '/report', {
'software_name': 'spam',
'software_version': '2.0',
'probe_asn': 'AS0',
'probe_cc': 'ZZ',
'test_name': 'foobar',
'test_version': '1.0',
'input_hashes': []
})
report_id = str(res['report_id'])
res = yield self.collector_client.queryBackend(
'POST', '/report/' + report_id,
{'content': '---\nspam: ham\n...\n'})
res = yield self.collector_client.queryBackend(
'POST', '/report/' + report_id,
{'content': '---\nspam: ham\n...\n'})
res = yield self.collector_client.queryBackend(
'POST', '/report/' + report_id + '/close')
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
try:
# Try to guess the measurement_id from the file path
measurement_id = report_path_to_id(report_file)
except NoIDFound:
pass
log.msg("Attempting to upload %s" % report_file)
if report_file.endswith(".njson"):
report = NJSONReportLoader(report_file)
else:
log.warn("Uploading of YAML formatted reports will be dropped in "
"future versions")
report = YAMLReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
if measurement_id:
report_log = yield oonib_report_log.get_report_log(measurement_id)
collector_settings = report_log['collector']
print(collector_settings)
if collector_settings is None or len(collector_settings) == 0:
log.warn("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
log.msg("Looking up collector with canonical bouncer." %
report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
if measurement_id:
log.debug("Marking it as created")
yield oonib_report_log.created(measurement_id,
collector_client.settings)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Written entry")
log.msg("Closing report")
yield oonib_reporter.finish()
if measurement_id:
log.debug("Closing log")
yield oonib_report_log.closed(measurement_id)
