Beispiel #1
0
class TestOONIBReportLog(unittest.TestCase):
    def setUp(self):
        self.report_log = OONIBReportLog('report_log')
        self.report_log.create_report_log()

    def tearDown(self):
        os.remove(self.report_log.file_name)

    @defer.inlineCallbacks
    def test_report_created(self):
        yield self.report_log.created("path_to_my_report.yaml",
                                      'httpo://foo.onion', 'someid')
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report.yaml" in report

    @defer.inlineCallbacks
    def test_concurrent_edit(self):
        d1 = self.report_log.created("path_to_my_report1.yaml",
                                     'httpo://foo.onion', 'someid1')
        d2 = self.report_log.created("path_to_my_report2.yaml",
                                     'httpo://foo.onion', 'someid2')
        yield defer.DeferredList([d1, d2])
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report1.yaml" in report
            assert "path_to_my_report2.yaml" in report

    @defer.inlineCallbacks
    def test_report_closed(self):
        yield self.report_log.created("path_to_my_report.yaml",
                                      'httpo://foo.onion', 'someid')
        yield self.report_log.closed("path_to_my_report.yaml")

        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report.yaml" not in report

    @defer.inlineCallbacks
    def test_report_creation_failed(self):
        yield self.report_log.creation_failed("path_to_my_report.yaml",
                                              'httpo://foo.onion')
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
        assert "path_to_my_report.yaml" in report
        assert report["path_to_my_report.yaml"]["status"] == "creation-failed"

    @defer.inlineCallbacks
    def test_list_reports(self):
        yield self.report_log.creation_failed("failed_report.yaml",
                                              'httpo://foo.onion')
        yield self.report_log.created("created_report.yaml",
                                      'httpo://foo.onion', 'XXXX')

        assert len(self.report_log.reports_in_progress) == 1
        assert len(self.report_log.reports_incomplete) == 0
        assert len(self.report_log.reports_to_upload) == 1
Beispiel #2
0
 def setUp(self):
     super(TestOONIBReportLog, self).setUp()
     self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
     self.config.measurements_directory = tempfile.mkdtemp()
     self.measurement_dir = os.path.join(self.config.measurements_directory,
                                         self.measurement_id)
     self.report_log_path = os.path.join(self.measurement_dir,
                                         'report_log.json')
     os.mkdir(self.measurement_dir)
     self.report_log = OONIBReportLog()
class TestOONIBReportLog(ConfigTestCase):

    def setUp(self):
        super(TestOONIBReportLog, self).setUp()
        self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
        self.config.measurements_directory = tempfile.mkdtemp()
        self.measurement_dir = os.path.join(
            self.config.measurements_directory,
            self.measurement_id
        )
        self.report_log_path = os.path.join(self.measurement_dir,
                                            'report_log.json')
        os.mkdir(self.measurement_dir)
        self.report_log = OONIBReportLog()

    def tearDown(self):
        shutil.rmtree(self.measurement_dir)
        super(TestOONIBReportLog, self).tearDown()

    @defer.inlineCallbacks
    def test_report_created(self):
        yield self.report_log.created(self.measurement_id, {})
        with open(self.report_log_path) as f:
            report = json.load(f)
            self.assertEqual(report['status'], 'created')

    @defer.inlineCallbacks
    def test_report_closed(self):
        yield self.report_log.created(self.measurement_id, {})
        yield self.report_log.closed(self.measurement_id)

        self.assertFalse(os.path.exists(self.report_log_path))

    @defer.inlineCallbacks
    def test_report_creation_failed(self):
        yield self.report_log.creation_failed(self.measurement_id, {})
        with open(self.report_log_path) as f:
            report = json.load(f)
        self.assertEqual(report["status"], "creation-failed")

    @defer.inlineCallbacks
    def test_list_reports_in_progress(self):
        yield self.report_log.created(self.measurement_id, {})
        in_progress = yield self.report_log.get_in_progress()
        incomplete = yield self.report_log.get_incomplete()
        self.assertEqual(len(incomplete), 0)
        self.assertEqual(len(in_progress), 1)

    @defer.inlineCallbacks
    def test_list_reports_to_upload(self):
        yield self.report_log.creation_failed(self.measurement_id, {})
        incomplete = yield self.report_log.get_incomplete()
        to_upload = yield self.report_log.get_to_upload()
        self.assertEqual(len(incomplete), 0)
        self.assertEqual(len(to_upload), 1)
Beispiel #4
0
class TestOONIBReportLog(ConfigTestCase):
    def setUp(self):
        super(TestOONIBReportLog, self).setUp()
        self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
        self.config.measurements_directory = tempfile.mkdtemp()
        self.measurement_dir = os.path.join(self.config.measurements_directory,
                                            self.measurement_id)
        self.report_log_path = os.path.join(self.measurement_dir,
                                            'report_log.json')
        os.mkdir(self.measurement_dir)
        self.report_log = OONIBReportLog()

    def tearDown(self):
        shutil.rmtree(self.measurement_dir)
        super(TestOONIBReportLog, self).tearDown()

    @defer.inlineCallbacks
    def test_report_created(self):
        yield self.report_log.created(self.measurement_id, {})
        with open(self.report_log_path) as f:
            report = json.load(f)
            self.assertEqual(report['status'], 'created')

    @defer.inlineCallbacks
    def test_report_closed(self):
        yield self.report_log.created(self.measurement_id, {})
        yield self.report_log.closed(self.measurement_id)

        self.assertFalse(os.path.exists(self.report_log_path))

    @defer.inlineCallbacks
    def test_report_creation_failed(self):
        yield self.report_log.creation_failed(self.measurement_id, {})
        with open(self.report_log_path) as f:
            report = json.load(f)
        self.assertEqual(report["status"], "creation-failed")

    @defer.inlineCallbacks
    def test_list_reports_in_progress(self):
        yield self.report_log.created(self.measurement_id, {})
        in_progress = yield self.report_log.get_in_progress()
        incomplete = yield self.report_log.get_incomplete()
        self.assertEqual(len(incomplete), 0)
        self.assertEqual(len(in_progress), 1)

    @defer.inlineCallbacks
    def test_list_reports_to_upload(self):
        yield self.report_log.creation_failed(self.measurement_id, {})
        incomplete = yield self.report_log.get_incomplete()
        to_upload = yield self.report_log.get_to_upload()
        self.assertEqual(len(incomplete), 0)
        self.assertEqual(len(to_upload), 1)
Beispiel #5
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()
    collector_client = None
    if collector:
        collector_client = CollectorClient(address=collector)

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and collector_client is None:
        collector_client = yield lookup_collector_client(report.header,
                                                         bouncer)

    if collector_client is None:
        try:
            collector_settings = report_log[report_file]['collector']
            if collector_settings is None:
                log.msg("Skipping uploading of %s since this measurement "
                        "was run by specifying no collector." %
                        report_file)
                defer.returnValue(None)
            elif isinstance(collector_settings, dict):
                collector_client = CollectorClient(settings=collector_settings)
            elif isinstance(collector_settings, str):
                collector_client = CollectorClient(address=collector_settings)
        except KeyError:
            log.msg("Could not find %s in reporting.yaml. Looking up "
                    "collector with canonical bouncer." % report_file)
            collector_client = yield lookup_collector_client(report.header,
                                                             canonical_bouncer)

    oonib_reporter = OONIBReporter(report.header, collector_client)
    log.msg("Creating report for %s with %s" % (report_file,
                                                collector_client.settings))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file,
                                   collector_client.settings,
                                   report_id)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        sys.stdout.write('.')
        sys.stdout.flush()
    log.msg("Closing report")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Beispiel #6
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()
    collector_client = None
    if collector:
        collector_client = CollectorClient(address=collector)

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and collector_client is None:
        collector_client = yield lookup_collector_client(
            report.header, bouncer)

    if collector_client is None:
        try:
            collector_settings = report_log[report_file]['collector']
            if collector_settings is None:
                log.msg("Skipping uploading of %s since this measurement "
                        "was run by specifying no collector." % report_file)
                defer.returnValue(None)
            elif isinstance(collector_settings, dict):
                collector_client = CollectorClient(settings=collector_settings)
            elif isinstance(collector_settings, str):
                collector_client = CollectorClient(address=collector_settings)
        except KeyError:
            log.msg("Could not find %s in reporting.yaml. Looking up "
                    "collector with canonical bouncer." % report_file)
            collector_client = yield lookup_collector_client(
                report.header, CANONICAL_BOUNCER_ONION)

    oonib_reporter = OONIBReporter(report.header, collector_client)
    log.msg("Creating report for %s with %s" %
            (report_file, collector_client.settings))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file, collector_client.settings,
                                   report_id)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        sys.stdout.write('.')
        sys.stdout.flush()
    log.msg("Closing report")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Beispiel #7
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and not collector:
        oonib_client = OONIBClient(bouncer)
        net_tests = [{
            'test-helpers': [],
            'input-hashes': report.header['input_hashes'],
            'name': report.header['test_name'],
            'version': report.header['test_version'],
        }]
        result = yield oonib_client.lookupTestCollector(
            net_tests
        )
        collector = str(result['net-tests'][0]['collector'])

    if collector is None:
        try:
            collector = report_log[report_file]['collector']
            if collector is None:
                raise KeyError
        except KeyError:
            raise Exception(
                "No collector or bouncer specified"
                " and collector not in report log."
            )

    oonib_reporter = OONIBReporter(report.header, collector)
    log.msg("Creating report for %s with %s" % (report_file, collector))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file, collector, report_id)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        sys.stdout.write('.')
        sys.stdout.flush()
    log.msg("Closing report")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Beispiel #8
0
def upload_all(collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()

    for report_file, value in oonib_report_log.reports_to_upload:
        try:
            yield upload(report_file, collector, bouncer)
        except Exception as exc:
            log.exception(exc)
Beispiel #9
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and not collector:
        oonib_client = OONIBClient(bouncer)
        net_tests = [{
            'test-helpers': [],
            'input-hashes': report.header['input_hashes'],
            'name': report.header['test_name'],
            'version': report.header['test_version'],
        }]
        result = yield oonib_client.lookupTestCollector(
            net_tests
        )
        collector = str(result['net-tests'][0]['collector'])

    if collector is None:
        try:
            collector = report_log[report_file]['collector']
            if collector is None:
                raise KeyError
        except KeyError:
            raise Exception(
                "No collector or bouncer specified"
                " and collector not in report log."
            )

    oonib_reporter = OONIBReporter(report.header, collector)
    log.msg("Creating report for %s with %s" % (report_file, collector))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file, collector, report_id)
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
    log.msg("Closing report.")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Beispiel #10
0
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
    oonib_report_log = OONIBReportLog()

    reports_to_upload = yield oonib_report_log.get_to_upload()
    for report_file, value in reports_to_upload:
        try:
            yield upload(report_file, collector, bouncer,
                         value['measurement_id'])
        except Exception as exc:
            log.exception(exc)

    if upload_incomplete:
        reports_to_upload = yield oonib_report_log.get_incomplete()
        for report_file, value in reports_to_upload:
            try:
                yield upload(report_file, collector, bouncer,
                             value['measurement_id'])
            except Exception as exc:
                log.exception(exc)
Beispiel #11
0
def upload_all(collector=None, bouncer=None, upload_incomplete=False):
    oonib_report_log = OONIBReportLog()

    reports_to_upload = yield oonib_report_log.get_to_upload()
    for report_file, value in reports_to_upload:
        try:
            yield upload(report_file, collector, bouncer,
                         value['measurement_id'])
        except Exception as exc:
            log.exception(exc)

    if upload_incomplete:
        reports_to_upload = yield oonib_report_log.get_incomplete()
        for report_file, value in reports_to_upload:
            try:
                yield upload(report_file, collector, bouncer,
                             value['measurement_id'])
            except Exception as exc:
                log.exception(exc)
Beispiel #12
0
def status():
    oonib_report_log = OONIBReportLog()

    reports_to_upload = yield oonib_report_log.get_to_upload()
    print("Reports to be uploaded")
    print("----------------------")
    for report_file, value in reports_to_upload:
        print_report(report_file, value)

    reports_in_progress = yield oonib_report_log.get_in_progress()
    print("Reports in progress")
    print("-------------------")
    for report_file, value in reports_in_progress:
        print_report(report_file, value)

    reports_incomplete = yield oonib_report_log.get_incomplete()
    print("Incomplete reports")
    print("------------------")
    for report_file, value in reports_incomplete:
        print_report(report_file, value)
Beispiel #13
0
def status():
    oonib_report_log = OONIBReportLog()

    reports_to_upload = yield oonib_report_log.get_to_upload()
    print("Reports to be uploaded")
    print("----------------------")
    for report_file, value in reports_to_upload:
        print_report(report_file, value)

    reports_in_progress = yield oonib_report_log.get_in_progress()
    print("Reports in progress")
    print("-------------------")
    for report_file, value in reports_in_progress:
        print_report(report_file, value)

    reports_incomplete = yield oonib_report_log.get_incomplete()
    print("Incomplete reports")
    print("------------------")
    for report_file, value in reports_incomplete:
        print_report(report_file, value)
class TestOONIBReportLog(unittest.TestCase):

    def setUp(self):
        self.report_log = OONIBReportLog('report_log')
        self.report_log.create_report_log()

    def tearDown(self):
        os.remove(self.report_log.file_name)

    @defer.inlineCallbacks
    def test_report_created(self):
        yield self.report_log.created("path_to_my_report.yaml",
                                             'httpo://foo.onion',
                                             'someid')
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report.yaml" in report

    @defer.inlineCallbacks
    def test_concurrent_edit(self):
        d1 = self.report_log.created("path_to_my_report1.yaml",
                                            'httpo://foo.onion',
                                            'someid1')
        d2 = self.report_log.created("path_to_my_report2.yaml",
                                            'httpo://foo.onion',
                                            'someid2')
        yield defer.DeferredList([d1, d2])
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report1.yaml" in report
            assert "path_to_my_report2.yaml" in report

    @defer.inlineCallbacks
    def test_report_closed(self):
        yield self.report_log.created("path_to_my_report.yaml",
                                             'httpo://foo.onion',
                                             'someid')
        yield self.report_log.closed("path_to_my_report.yaml")

        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
            assert "path_to_my_report.yaml" not in report

    @defer.inlineCallbacks
    def test_report_creation_failed(self):
        yield self.report_log.creation_failed("path_to_my_report.yaml",
                                                     'httpo://foo.onion')
        with open(self.report_log.file_name) as f:
            report = yaml.safe_load(f)
        assert "path_to_my_report.yaml" in report
        assert report["path_to_my_report.yaml"]["status"] == "creation-failed"
 def setUp(self):
     super(TestOONIBReportLog, self).setUp()
     self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy'
     self.config.measurements_directory = tempfile.mkdtemp()
     self.measurement_dir = os.path.join(
         self.config.measurements_directory,
         self.measurement_id
     )
     self.report_log_path = os.path.join(self.measurement_dir,
                                         'report_log.json')
     os.mkdir(self.measurement_dir)
     self.report_log = OONIBReportLog()
Beispiel #16
0
def status():
    oonib_report_log = OONIBReportLog()

    print "Reports to be uploaded"
    print "----------------------"
    for report_file, value in oonib_report_log.reports_to_upload:
        print_report(report_file, value)

    print "Reports in progress"
    print "-------------------"
    for report_file, value in oonib_report_log.reports_in_progress:
        print_report(report_file, value)

    print "Incomplete reports"
    print "------------------"
    for report_file, value in oonib_report_log.reports_incomplete:
        print_report(report_file, value)
Beispiel #17
0
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
    oonib_report_log = OONIBReportLog()
    collector_client = None
    if collector:
        collector_client = CollectorClient(address=collector)

    try:
        # Try to guess the measurement_id from the file path
        measurement_id = report_path_to_id(report_file)
    except NoIDFound:
        pass

    log.msg("Attempting to upload %s" % report_file)

    if report_file.endswith(".njson"):
        report = NJSONReportLoader(report_file)
    else:
        log.warn("Uploading of YAML formatted reports will be dropped in "
                 "future versions")
        report = YAMLReportLoader(report_file)

    if bouncer and collector_client is None:
        collector_client = yield lookup_collector_client(
            report.header, bouncer)

    if collector_client is None:
        if measurement_id:
            report_log = yield oonib_report_log.get_report_log(measurement_id)
            collector_settings = report_log['collector']
            print(collector_settings)
            if collector_settings is None or len(collector_settings) == 0:
                log.warn("Skipping uploading of %s since this measurement "
                         "was run by specifying no collector." % report_file)
                defer.returnValue(None)
            elif isinstance(collector_settings, dict):
                collector_client = CollectorClient(settings=collector_settings)
            elif isinstance(collector_settings, str):
                collector_client = CollectorClient(address=collector_settings)
        else:
            log.msg("Looking up collector with canonical bouncer." %
                    report_file)
            collector_client = yield lookup_collector_client(
                report.header, CANONICAL_BOUNCER_ONION)

    oonib_reporter = OONIBReporter(report.header, collector_client)
    log.msg("Creating report for %s with %s" %
            (report_file, collector_client.settings))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    if measurement_id:
        log.debug("Marking it as created")
        yield oonib_report_log.created(measurement_id,
                                       collector_client.settings)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        log.msg("Written entry")
    log.msg("Closing report")
    yield oonib_reporter.finish()
    if measurement_id:
        log.debug("Closing log")
        yield oonib_report_log.closed(measurement_id)
Beispiel #18
0
def upload(report_file, collector=None, bouncer=None, measurement_id=None):
    oonib_report_log = OONIBReportLog()
    collector_client = None
    if collector:
        collector_client = CollectorClient(address=collector)

    try:
        # Try to guess the measurement_id from the file path
        measurement_id = report_path_to_id(report_file)
    except NoIDFound:
        pass

    log.msg("Attempting to upload %s" % report_file)

    if report_file.endswith(".njson"):
        report = NJSONReportLoader(report_file)
    else:
        log.warn("Uploading of YAML formatted reports will be dropped in "
                 "future versions")
        report = YAMLReportLoader(report_file)

    if bouncer and collector_client is None:
        collector_client = yield lookup_collector_client(report.header,
                                                         bouncer)

    if collector_client is None:
        if measurement_id:
            report_log = yield oonib_report_log.get_report_log(measurement_id)
            collector_settings = report_log['collector']
            print(collector_settings)
            if collector_settings is None or len(collector_settings) == 0:
                log.warn("Skipping uploading of %s since this measurement "
                         "was run by specifying no collector." %
                          report_file)
                defer.returnValue(None)
            elif isinstance(collector_settings, dict):
                collector_client = CollectorClient(settings=collector_settings)
            elif isinstance(collector_settings, str):
                collector_client = CollectorClient(address=collector_settings)
        else:
            log.msg("Looking up collector with canonical bouncer." % report_file)
            collector_client = yield lookup_collector_client(report.header,
                                                             CANONICAL_BOUNCER_ONION)

    oonib_reporter = OONIBReporter(report.header, collector_client)
    log.msg("Creating report for %s with %s" % (report_file,
                                                collector_client.settings))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    if measurement_id:
        log.debug("Marking it as created")
        yield oonib_report_log.created(measurement_id,
                                       collector_client.settings)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        log.msg("Written entry")
    log.msg("Closing report")
    yield oonib_reporter.finish()
    if measurement_id:
        log.debug("Closing log")
        yield oonib_report_log.closed(measurement_id)
Beispiel #19
0
 def setUp(self):
     self.report_log = OONIBReportLog('report_log')
     self.report_log.create_report_log()
Beispiel #20
0
 def setUp(self):
     self.report_log = OONIBReportLog('report_log')
     self.report_log.create_report_log()