def upload_all(collector=None, bouncer=None): oonib_report_log = OONIBReportLog() for report_file, value in oonib_report_log.reports_to_upload: try: yield upload(report_file, collector, bouncer) except Exception as exc: log.exception(exc)
def setUp(self): super(TestOONIBReportLog, self).setUp() self.measurement_id = '20160727T182604Z-ZZ-AS0-dummy' self.config.measurements_directory = tempfile.mkdtemp() self.measurement_dir = os.path.join(self.config.measurements_directory, self.measurement_id) self.report_log_path = os.path.join(self.measurement_dir, 'report_log.json') os.mkdir(self.measurement_dir) self.report_log = OONIBReportLog()
def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() collector_client = None if collector: collector_client = CollectorClient(address=collector) log.msg("Attempting to upload %s" % report_file) with open(config.report_log_file) as f: report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) if bouncer and collector_client is None: collector_client = yield lookup_collector_client( report.header, bouncer) if collector_client is None: try: collector_settings = report_log[report_file]['collector'] if collector_settings is None: log.msg("Skipping uploading of %s since this measurement " "was run by specifying no collector." % report_file) defer.returnValue(None) elif isinstance(collector_settings, dict): collector_client = CollectorClient(settings=collector_settings) elif isinstance(collector_settings, str): collector_client = CollectorClient(address=collector_settings) except KeyError: log.msg("Could not find %s in reporting.yaml. Looking up " "collector with canonical bouncer." % report_file) collector_client = yield lookup_collector_client( report.header, CANONICAL_BOUNCER_ONION) oonib_reporter = OONIBReporter(report.header, collector_client) log.msg("Creating report for %s with %s" % (report_file, collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector_client.settings, report_id) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) sys.stdout.write('.') sys.stdout.flush() log.msg("Closing report") yield oonib_reporter.finish() yield oonib_report_log.closed(report_file)
def status(): oonib_report_log = OONIBReportLog() print("Reports to be uploaded") print("----------------------") for report_file, value in oonib_report_log.reports_to_upload: print_report(report_file, value) print("Reports in progress") print("-------------------") for report_file, value in oonib_report_log.reports_in_progress: print_report(report_file, value) print("Incomplete reports") print("------------------") for report_file, value in oonib_report_log.reports_incomplete: print_report(report_file, value)
def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() log.msg("Attempting to upload %s" % report_file) with open(config.report_log_file) as f: report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) if bouncer and not collector: oonib_client = OONIBClient(bouncer) net_tests = [{ 'test-helpers': [], 'input-hashes': report.header['input_hashes'], 'name': report.header['test_name'], 'version': report.header['test_version'], }] result = yield oonib_client.lookupTestCollector( net_tests ) collector = str(result['net-tests'][0]['collector']) if collector is None: try: collector = report_log[report_file]['collector'] if collector is None: raise KeyError except KeyError: raise Exception( "No collector or bouncer specified" " and collector not in report log." ) oonib_reporter = OONIBReporter(report.header, collector) log.msg("Creating report for %s with %s" % (report_file, collector)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector, report_id) for entry in report: yield oonib_reporter.writeReportEntry(entry) log.msg("Closing report.") yield oonib_reporter.finish() yield oonib_report_log.closed(report_file)
def upload_all(collector=None, bouncer=None, upload_incomplete=False): oonib_report_log = OONIBReportLog() reports_to_upload = yield oonib_report_log.get_to_upload() for report_file, value in reports_to_upload: try: yield upload(report_file, collector, bouncer, value['measurement_id']) except Exception as exc: log.exception(exc) if upload_incomplete: reports_to_upload = yield oonib_report_log.get_incomplete() for report_file, value in reports_to_upload: try: yield upload(report_file, collector, bouncer, value['measurement_id']) except Exception as exc: log.exception(exc)
def upload(report_file, collector=None, bouncer=None, measurement_id=None): oonib_report_log = OONIBReportLog() collector_client = None if collector: collector_client = CollectorClient(address=collector) try: # Try to guess the measurement_id from the file path measurement_id = report_path_to_id(report_file) except NoIDFound: pass log.msg("Attempting to upload %s" % report_file) if report_file.endswith(".njson"): report = NJSONReportLoader(report_file) else: log.warn("Uploading of YAML formatted reports will be dropped in " "future versions") report = YAMLReportLoader(report_file) if bouncer and collector_client is None: collector_client = yield lookup_collector_client( report.header, bouncer) if collector_client is None: if measurement_id: report_log = yield oonib_report_log.get_report_log(measurement_id) collector_settings = report_log['collector'] print(collector_settings) if collector_settings is None or len(collector_settings) == 0: log.warn("Skipping uploading of %s since this measurement " "was run by specifying no collector." % report_file) defer.returnValue(None) elif isinstance(collector_settings, dict): collector_client = CollectorClient(settings=collector_settings) elif isinstance(collector_settings, str): collector_client = CollectorClient(address=collector_settings) else: log.msg("Looking up collector with canonical bouncer." % report_file) collector_client = yield lookup_collector_client( report.header, CANONICAL_BOUNCER_ONION) oonib_reporter = OONIBReporter(report.header, collector_client) log.msg("Creating report for %s with %s" % (report_file, collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id if measurement_id: log.debug("Marking it as created") yield oonib_report_log.created(measurement_id, collector_client.settings) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) log.msg("Written entry") log.msg("Closing report") yield oonib_reporter.finish() if measurement_id: log.debug("Closing log") yield oonib_report_log.closed(measurement_id)
def setUp(self): self.report_log = OONIBReportLog('report_log') self.report_log.create_report_log()