class TestOONIBReportLog(unittest.TestCase):
def setUp(self):
self.report_log = OONIBReportLog('report_log')
self.report_log.create_report_log()
def tearDown(self):
os.remove(self.report_log.file_name)
@defer.inlineCallbacks
def test_report_created(self):
yield self.report_log.created("path_to_my_report.yaml",
'httpo://foo.onion', 'someid')
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" in report
@defer.inlineCallbacks
def test_concurrent_edit(self):
d1 = self.report_log.created("path_to_my_report1.yaml",
'httpo://foo.onion', 'someid1')
d2 = self.report_log.created("path_to_my_report2.yaml",
'httpo://foo.onion', 'someid2')
yield defer.DeferredList([d1, d2])
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report1.yaml" in report
assert "path_to_my_report2.yaml" in report
@defer.inlineCallbacks
def test_report_closed(self):
yield self.report_log.created("path_to_my_report.yaml",
'httpo://foo.onion', 'someid')
yield self.report_log.closed("path_to_my_report.yaml")
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" not in report
@defer.inlineCallbacks
def test_report_creation_failed(self):
yield self.report_log.creation_failed("path_to_my_report.yaml",
'httpo://foo.onion')
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" in report
assert report["path_to_my_report.yaml"]["status"] == "creation-failed"
@defer.inlineCallbacks
def test_list_reports(self):
yield self.report_log.creation_failed("failed_report.yaml",
'httpo://foo.onion')
yield self.report_log.created("created_report.yaml",
'httpo://foo.onion', 'XXXX')
assert len(self.report_log.reports_in_progress) == 1
assert len(self.report_log.reports_incomplete) == 0
assert len(self.report_log.reports_to_upload) == 1
def setUp(self):
super(TestOONIBReportLog, self).setUp()
self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
self.config.measurements_directory = tempfile.mkdtemp()
self.measurement_dir = os.path.join(self.config.measurements_directory,
self.measurement_id)
self.report_log_path = os.path.join(self.measurement_dir,
'report_log.json')
os.mkdir(self.measurement_dir)
self.report_log = OONIBReportLog()
class TestOONIBReportLog(ConfigTestCase):
def setUp(self):
super(TestOONIBReportLog, self).setUp()
self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
self.config.measurements_directory = tempfile.mkdtemp()
self.measurement_dir = os.path.join(
self.config.measurements_directory,
self.measurement_id
)
self.report_log_path = os.path.join(self.measurement_dir,
'report_log.json')
os.mkdir(self.measurement_dir)
self.report_log = OONIBReportLog()
def tearDown(self):
shutil.rmtree(self.measurement_dir)
super(TestOONIBReportLog, self).tearDown()
@defer.inlineCallbacks
def test_report_created(self):
yield self.report_log.created(self.measurement_id, {})
with open(self.report_log_path) as f:
report = json.load(f)
self.assertEqual(report['status'], 'created')
@defer.inlineCallbacks
def test_report_closed(self):
yield self.report_log.created(self.measurement_id, {})
yield self.report_log.closed(self.measurement_id)
self.assertFalse(os.path.exists(self.report_log_path))
@defer.inlineCallbacks
def test_report_creation_failed(self):
yield self.report_log.creation_failed(self.measurement_id, {})
with open(self.report_log_path) as f:
report = json.load(f)
self.assertEqual(report["status"], "creation-failed")
@defer.inlineCallbacks
def test_list_reports_in_progress(self):
yield self.report_log.created(self.measurement_id, {})
in_progress = yield self.report_log.get_in_progress()
incomplete = yield self.report_log.get_incomplete()
self.assertEqual(len(incomplete), 0)
self.assertEqual(len(in_progress), 1)
@defer.inlineCallbacks
def test_list_reports_to_upload(self):
yield self.report_log.creation_failed(self.measurement_id, {})
incomplete = yield self.report_log.get_incomplete()
to_upload = yield self.report_log.get_to_upload()
self.assertEqual(len(incomplete), 0)
self.assertEqual(len(to_upload), 1)
class TestOONIBReportLog(ConfigTestCase):
def setUp(self):
super(TestOONIBReportLog, self).setUp()
self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
self.config.measurements_directory = tempfile.mkdtemp()
self.measurement_dir = os.path.join(self.config.measurements_directory,
self.measurement_id)
self.report_log_path = os.path.join(self.measurement_dir,
'report_log.json')
os.mkdir(self.measurement_dir)
self.report_log = OONIBReportLog()
def tearDown(self):
shutil.rmtree(self.measurement_dir)
super(TestOONIBReportLog, self).tearDown()
@defer.inlineCallbacks
def test_report_created(self):
yield self.report_log.created(self.measurement_id, {})
with open(self.report_log_path) as f:
report = json.load(f)
self.assertEqual(report['status'], 'created')
@defer.inlineCallbacks
def test_report_closed(self):
yield self.report_log.created(self.measurement_id, {})
yield self.report_log.closed(self.measurement_id)
self.assertFalse(os.path.exists(self.report_log_path))
@defer.inlineCallbacks
def test_report_creation_failed(self):
yield self.report_log.creation_failed(self.measurement_id, {})
with open(self.report_log_path) as f:
report = json.load(f)
self.assertEqual(report["status"], "creation-failed")
@defer.inlineCallbacks
def test_list_reports_in_progress(self):
yield self.report_log.created(self.measurement_id, {})
in_progress = yield self.report_log.get_in_progress()
incomplete = yield self.report_log.get_incomplete()
self.assertEqual(len(incomplete), 0)
self.assertEqual(len(in_progress), 1)
@defer.inlineCallbacks
def test_list_reports_to_upload(self):
yield self.report_log.creation_failed(self.measurement_id, {})
incomplete = yield self.report_log.get_incomplete()
to_upload = yield self.report_log.get_to_upload()
self.assertEqual(len(incomplete), 0)
self.assertEqual(len(to_upload), 1)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(report.header,
bouncer)
if collector_client is None:
try:
collector_settings = report_log[report_file]['collector']
if collector_settings is None:
log.msg("Skipping uploading of %s since this measurement "
"was run by specifying no collector." %
report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
except KeyError:
log.msg("Could not find %s in reporting.yaml. Looking up "
"collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(report.header,
canonical_bouncer)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" % (report_file,
collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file,
collector_client.settings,
report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
try:
collector_settings = report_log[report_file]['collector']
if collector_settings is None:
log.msg("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
except KeyError:
log.msg("Could not find %s in reporting.yaml. Looking up "
"collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector_client.settings,
report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload_all(collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
for report_file, value in oonib_report_log.reports_to_upload:
try:
yield upload(report_file, collector, bouncer)
except Exception as exc:
log.exception(exc)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Closing report.")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
if upload_incomplete:
reports_to_upload = yield oonib_report_log.get_incomplete()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
if upload_incomplete:
reports_to_upload = yield oonib_report_log.get_incomplete()
for report_file, value in reports_to_upload:
try:
yield upload(report_file, collector, bouncer,
value['measurement_id'])
except Exception as exc:
log.exception(exc)
def status():
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
print("Reports to be uploaded")
print("----------------------")
for report_file, value in reports_to_upload:
print_report(report_file, value)
reports_in_progress = yield oonib_report_log.get_in_progress()
print("Reports in progress")
print("-------------------")
for report_file, value in reports_in_progress:
print_report(report_file, value)
reports_incomplete = yield oonib_report_log.get_incomplete()
print("Incomplete reports")
print("------------------")
for report_file, value in reports_incomplete:
print_report(report_file, value)
def status():
oonib_report_log = OONIBReportLog()
reports_to_upload = yield oonib_report_log.get_to_upload()
print("Reports to be uploaded")
print("----------------------")
for report_file, value in reports_to_upload:
print_report(report_file, value)
reports_in_progress = yield oonib_report_log.get_in_progress()
print("Reports in progress")
print("-------------------")
for report_file, value in reports_in_progress:
print_report(report_file, value)
reports_incomplete = yield oonib_report_log.get_incomplete()
print("Incomplete reports")
print("------------------")
for report_file, value in reports_incomplete:
print_report(report_file, value)
class TestOONIBReportLog(unittest.TestCase):
def setUp(self):
self.report_log = OONIBReportLog('report_log')
self.report_log.create_report_log()
def tearDown(self):
os.remove(self.report_log.file_name)
@defer.inlineCallbacks
def test_report_created(self):
yield self.report_log.created("path_to_my_report.yaml",
'httpo://foo.onion',
'someid')
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" in report
@defer.inlineCallbacks
def test_concurrent_edit(self):
d1 = self.report_log.created("path_to_my_report1.yaml",
'httpo://foo.onion',
'someid1')
d2 = self.report_log.created("path_to_my_report2.yaml",
'httpo://foo.onion',
'someid2')
yield defer.DeferredList([d1, d2])
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report1.yaml" in report
assert "path_to_my_report2.yaml" in report
@defer.inlineCallbacks
def test_report_closed(self):
yield self.report_log.created("path_to_my_report.yaml",
'httpo://foo.onion',
'someid')
yield self.report_log.closed("path_to_my_report.yaml")
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" not in report
@defer.inlineCallbacks
def test_report_creation_failed(self):
yield self.report_log.creation_failed("path_to_my_report.yaml",
'httpo://foo.onion')
with open(self.report_log.file_name) as f:
report = yaml.safe_load(f)
assert "path_to_my_report.yaml" in report
assert report["path_to_my_report.yaml"]["status"] == "creation-failed"
def setUp(self):
super(TestOONIBReportLog, self).setUp()
self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
self.config.measurements_directory = tempfile.mkdtemp()
self.measurement_dir = os.path.join(
self.config.measurements_directory,
self.measurement_id
)
self.report_log_path = os.path.join(self.measurement_dir,
'report_log.json')
os.mkdir(self.measurement_dir)
self.report_log = OONIBReportLog()
def status():
oonib_report_log = OONIBReportLog()
print "Reports to be uploaded"
print "----------------------"
for report_file, value in oonib_report_log.reports_to_upload:
print_report(report_file, value)
print "Reports in progress"
print "-------------------"
for report_file, value in oonib_report_log.reports_in_progress:
print_report(report_file, value)
print "Incomplete reports"
print "------------------"
for report_file, value in oonib_report_log.reports_incomplete:
print_report(report_file, value)
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
try:
# Try to guess the measurement_id from the file path
measurement_id = report_path_to_id(report_file)
except NoIDFound:
pass
log.msg("Attempting to upload %s" % report_file)
if report_file.endswith(".njson"):
report = NJSONReportLoader(report_file)
else:
log.warn("Uploading of YAML formatted reports will be dropped in "
"future versions")
report = YAMLReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(
report.header, bouncer)
if collector_client is None:
if measurement_id:
report_log = yield oonib_report_log.get_report_log(measurement_id)
collector_settings = report_log['collector']
print(collector_settings)
if collector_settings is None or len(collector_settings) == 0:
log.warn("Skipping uploading of %s since this measurement "
"was run by specifying no collector." % report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
log.msg("Looking up collector with canonical bouncer." %
report_file)
collector_client = yield lookup_collector_client(
report.header, CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" %
(report_file, collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
if measurement_id:
log.debug("Marking it as created")
yield oonib_report_log.created(measurement_id,
collector_client.settings)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Written entry")
log.msg("Closing report")
yield oonib_reporter.finish()
if measurement_id:
log.debug("Closing log")
yield oonib_report_log.closed(measurement_id)
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
oonib_report_log = OONIBReportLog()
collector_client = None
if collector:
collector_client = CollectorClient(address=collector)
try:
# Try to guess the measurement_id from the file path
measurement_id = report_path_to_id(report_file)
except NoIDFound:
pass
log.msg("Attempting to upload %s" % report_file)
if report_file.endswith(".njson"):
report = NJSONReportLoader(report_file)
else:
log.warn("Uploading of YAML formatted reports will be dropped in "
"future versions")
report = YAMLReportLoader(report_file)
if bouncer and collector_client is None:
collector_client = yield lookup_collector_client(report.header,
bouncer)
if collector_client is None:
if measurement_id:
report_log = yield oonib_report_log.get_report_log(measurement_id)
collector_settings = report_log['collector']
print(collector_settings)
if collector_settings is None or len(collector_settings) == 0:
log.warn("Skipping uploading of %s since this measurement "
"was run by specifying no collector." %
report_file)
defer.returnValue(None)
elif isinstance(collector_settings, dict):
collector_client = CollectorClient(settings=collector_settings)
elif isinstance(collector_settings, str):
collector_client = CollectorClient(address=collector_settings)
else:
log.msg("Looking up collector with canonical bouncer." % report_file)
collector_client = yield lookup_collector_client(report.header,
CANONICAL_BOUNCER_ONION)
oonib_reporter = OONIBReporter(report.header, collector_client)
log.msg("Creating report for %s with %s" % (report_file,
collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
if measurement_id:
log.debug("Marking it as created")
yield oonib_report_log.created(measurement_id,
collector_client.settings)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Written entry")
log.msg("Closing report")
yield oonib_reporter.finish()
if measurement_id:
log.debug("Closing log")
yield oonib_report_log.closed(measurement_id)
def setUp(self):
self.report_log = OONIBReportLog('report_log')
self.report_log.create_report_log()
def setUp(self):
self.report_log = OONIBReportLog('report_log')
self.report_log.create_report_log()
