def upload_all(collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
for report_file, value in oonib_report_log.reports_to_upload:
try:
yield upload(report_file, collector, bouncer)
except Exception as exc:
log.exception(exc)
def setUp(self):
super(TestOONIBReportLog, self).setUp()
self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
self.config.measurements_directory = tempfile.mkdtemp()
self.measurement_dir = os.path.join(self.config.measurements_directory,
self.measurement_id)
self.report_log_path = os.path.join(self.measurement_dir,
'report_log.json')
os.mkdir(self.measurement_dir)
self.report_log = OONIBReportLog()
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
try:
collector_settings = report_log[report_file]['collector']
if collector_settings is None:
log.msg("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
except KeyError:
log.msg("Could not find %s in reporting.yaml. Looking up "
"collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector_client.settings,
report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def status():
oonib_report_log = OONIBReportLog()
print("Reports to be uploaded")
print("----------------------")
for report_file, value in oonib_report_log.reports_to_upload:
print_report(report_file, value)
print("Reports in progress")
print("-------------------")
for report_file, value in oonib_report_log.reports_in_progress:
print_report(report_file, value)
print("Incomplete reports")
print("------------------")
for report_file, value in oonib_report_log.reports_incomplete:
print_report(report_file, value)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Closing report.")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
if upload_incomplete:
reports_to_upload = yield oonib_report_log.get_incomplete()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
try:
# Try to guess the measurement_id from the file path
measurement_id = report_path_to_id(report_file)
except NoIDFound:
pass
log.msg("Attempting to upload %s" % report_file)
if report_file.endswith(".njson"):
report = NJSONReportLoader(report_file)
else:
log.warn("Uploading of YAML formatted reports will be dropped in "
"future versions")
report = YAMLReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
if measurement_id:
report_log = yield oonib_report_log.get_report_log(measurement_id)
collector_settings = report_log['collector']
print(collector_settings)
if collector_settings is None or len(collector_settings) == 0:
log.warn("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
log.msg("Looking up collector with canonical bouncer." %
report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
if measurement_id:
log.debug("Marking it as created")
yield oonib_report_log.created(measurement_id,
collector_client.settings)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Written entry")
log.msg("Closing report")
yield oonib_reporter.finish()
if measurement_id:
log.debug("Closing log")
yield oonib_report_log.closed(measurement_id)
def setUp(self):
self.report_log = OONIBReportLog('report_log')
self.report_log.create_report_log()