def setUp(self):
self.mock_response = {}
def mockRequest(method, urn, genReceiver, *args, **kw):
receiver = genReceiver(None, None)
return defer.maybeDeferred(receiver.body_processor,
json.dumps(self.mock_response))
mock_collector_client = MockCollectorClient('http://example.com')
mock_collector_client._request = mockRequest
self.oonib_reporter = OONIBReporter(test_details,
mock_collector_client)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
try:
collector_settings = report_log[report_file]['collector']
if collector_settings is None:
log.msg("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
except KeyError:
log.msg("Could not find %s in reporting.yaml. Looking up "
"collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector_client.settings,
report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def setUp(self):
self.mock_response = {}
self.collector_address = 'http://example.com'
self.oonib_reporter = OONIBReporter(test_details,
self.collector_address)
self.oonib_reporter.agent = MagicMock()
self.mock_agent_response = MagicMock()
def deliverBody(body_receiver):
body_receiver.dataReceived(json.dumps(self.mock_response))
body_receiver.connectionLost(None)
self.mock_agent_response.deliverBody = deliverBody
self.oonib_reporter.agent.request.return_value = defer.succeed(
self.mock_agent_response)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Closing report.")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def post_director_start(_):
for net_test_loader in test_list:
collector = global_options['collector']
test_details = net_test_loader.testDetails
yaml_reporter = YAMLReporter(test_details)
reporters = [yaml_reporter]
if collector and collector.startswith('httpo') \
and (not (config.tor_state or config.tor.socks_port)):
raise errors.TorNotRunning
elif collector:
log.msg("Reporting using collector: %s" % collector)
try:
oonib_reporter = OONIBReporter(test_details, collector)
reporters.append(oonib_reporter)
except errors.InvalidOONIBCollectorAddress, e:
raise e
log.debug("adding callback for startNetTest")
d.addCallback(director.startNetTest, net_test_loader, reporters)
def get_reporters(net_test_loader):
"""
Determines which reports are able to run and returns an instance of them.
We always report to flat file via the :class:`ooni.reporters.YAMLReporter`
and the :class:`ooni.reporters.OONIBReporter`.
The later will be used only if we determine that Tor is running.
Returns:
a list of reporter instances
"""
test_details = net_test_loader.testDetails
reporters = []
yaml_reporter = YAMLReporter(test_details, config.reports_directory)
reporters.append(yaml_reporter)
if config.reports.collector and collector_supported(config.reports.collector):
oonib_reporter = OONIBReporter(test_details, config.reports.collector)
reporters.append(oonib_reporter)
return reporters
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
try:
# Try to guess the measurement_id from the file path
measurement_id = report_path_to_id(report_file)
except NoIDFound:
pass
log.msg("Attempting to upload %s" % report_file)
if report_file.endswith(".njson"):
report = NJSONReportLoader(report_file)
else:
log.warn("Uploading of YAML formatted reports will be dropped in "
"future versions")
report = YAMLReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
if measurement_id:
report_log = yield oonib_report_log.get_report_log(measurement_id)
collector_settings = report_log['collector']
print(collector_settings)
if collector_settings is None or len(collector_settings) == 0:
log.warn("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
log.msg("Looking up collector with canonical bouncer." %
report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
if measurement_id:
log.debug("Marking it as created")
yield oonib_report_log.created(measurement_id,
collector_client.settings)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Written entry")
log.msg("Closing report")
yield oonib_reporter.finish()
if measurement_id:
log.debug("Closing log")
yield oonib_report_log.closed(measurement_id)
