def upload(report_file, collector=None, bouncer=None):
print "Attempting to upload %s" % report_file
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer:
oonib_client = OONIBClient(bouncer)
collector = yield oonib_client.lookupTestCollector(report.header["test_name"])
if collector is None:
try:
collector = report_log[report_file]["collector"]
if collector is None:
raise KeyError
except KeyError:
raise Exception("No collector or bouncer specified and collector not in report log.")
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
yield oonib_report_log.created(report_file, collector, report_id)
for entry in report:
print "Writing entry"
yield oonib_reporter.writeReportEntry(entry)
log.msg("Closing report.")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
sys.stdout.write('.')
sys.stdout.flush()
log.msg("Closing report")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
log.msg("Attempting to upload %s" % report_file)
with open(config.report_log_file) as f:
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
if bouncer and not collector:
oonib_client = OONIBClient(bouncer)
net_tests = [{
'test-helpers': [],
'input-hashes': report.header['input_hashes'],
'name': report.header['test_name'],
'version': report.header['test_version'],
}]
result = yield oonib_client.lookupTestCollector(
net_tests
)
collector = str(result['net-tests'][0]['collector'])
if collector is None:
try:
collector = report_log[report_file]['collector']
if collector is None:
raise KeyError
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
oonib_reporter = OONIBReporter(report.header, collector)
log.msg("Creating report for %s with %s" % (report_file, collector))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
yield oonib_report_log.created(report_file, collector, report_id)
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
log.msg("Closing report.")
yield oonib_reporter.finish()
yield oonib_report_log.closed(report_file)
class Deck(InputFile):
def __init__(self, deck_hash=None,
deckFile=None,
decks_directory=config.decks_directory,
no_collector=False):
self.id = deck_hash
self.requiresTor = False
self.no_collector = no_collector
self.bouncer = ''
self.netTestLoaders = []
self.inputs = []
self.oonibclient = OONIBClient(self.bouncer)
self.decksDirectory = os.path.abspath(decks_directory)
self.deckHash = deck_hash
if deckFile:
self.loadDeck(deckFile)
@property
def cached_file(self):
return os.path.join(self.decksDirectory, self.deckHash)
@property
def cached_descriptor(self):
return self.cached_file + '.desc'
def loadDeck(self, deckFile):
with open(deckFile) as f:
self.deckHash = sha256(f.read()).hexdigest()
f.seek(0)
test_deck = yaml.safe_load(f)
for test in test_deck:
try:
nettest_path = nettest_to_path(test['options']['test_file'])
except e.NetTestNotFound:
log.err("Could not find %s" % test['options']['test_file'])
log.msg("Skipping...")
continue
net_test_loader = NetTestLoader(test['options']['subargs'],
test_file=nettest_path)
if test['options']['collector']:
net_test_loader.collector = test['options']['collector']
self.insert(net_test_loader)
def insert(self, net_test_loader):
""" Add a NetTestLoader to this test deck """
def has_test_helper(missing_option):
for rth in net_test_loader.requiredTestHelpers:
if missing_option == rth['option']:
return True
return False
try:
net_test_loader.checkOptions()
if net_test_loader.requiresTor:
self.requiresTor = True
except e.MissingRequiredOption as missing_options:
if not self.bouncer:
raise
for missing_option in missing_options.message:
if not has_test_helper(missing_option):
raise
self.requiresTor = True
self.netTestLoaders.append(net_test_loader)
@defer.inlineCallbacks
def setup(self):
""" fetch and verify inputs for all NetTests in the deck """
log.msg("Fetching required net test inputs...")
for net_test_loader in self.netTestLoaders:
yield self.fetchAndVerifyNetTestInput(net_test_loader)
if self.bouncer:
log.msg("Looking up collector and test helpers")
yield self.lookupCollector()
@defer.inlineCallbacks
def lookupCollector(self):
self.oonibclient.address = self.bouncer
required_nettests = []
requires_test_helpers = False
requires_collector = False
for net_test_loader in self.netTestLoaders:
nettest = {
'name': net_test_loader.testDetails['test_name'],
'version': net_test_loader.testDetails['test_version'],
'test-helpers': [],
'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
}
if not net_test_loader.collector and not self.no_collector:
requires_collector = True
for th in net_test_loader.requiredTestHelpers:
# {'name':'', 'option':'', 'test_class':''}
if th['test_class'].localOptions[th['option']]:
continue
nettest['test-helpers'].append(th['name'])
requires_test_helpers = True
required_nettests.append(nettest)
if not requires_test_helpers and not requires_collector:
defer.returnValue(None)
response = yield self.oonibclient.lookupTestCollector(required_nettests)
provided_net_tests = response['net-tests']
def find_collector_and_test_helpers(test_name, test_version, input_files):
input_files = [u""+x['hash'] for x in input_files]
for net_test in provided_net_tests:
if net_test['name'] != test_name:
continue
if net_test['version'] != test_version:
continue
if set(net_test['input-hashes']) != set(input_files):
continue
return net_test['collector'], net_test['test-helpers']
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])
collector, test_helpers = \
find_collector_and_test_helpers(net_test_loader.testDetails['test_name'],
net_test_loader.testDetails['test_version'],
net_test_loader.inputFiles)
for th in net_test_loader.requiredTestHelpers:
if not th['test_class'].localOptions[th['option']]:
th['test_class'].localOptions[th['option']] = test_helpers[th['name']].encode('utf-8')
net_test_loader.testHelpers[th['option']] = th['test_class'].localOptions[th['option']]
if not net_test_loader.collector:
net_test_loader.collector = collector.encode('utf-8')
@defer.inlineCallbacks
def lookupTestHelpers(self):
self.oonibclient.address = self.bouncer
required_test_helpers = []
requires_collector = []
for net_test_loader in self.netTestLoaders:
if not net_test_loader.collector and not self.no_collector:
requires_collector.append(net_test_loader)
for th in net_test_loader.requiredTestHelpers:
# {'name':'', 'option':'', 'test_class':''}
if th['test_class'].localOptions[th['option']]:
continue
required_test_helpers.append(th['name'])
if not required_test_helpers and not requires_collector:
defer.returnValue(None)
response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" %
net_test_loader.testDetails['test_name'])
# Only set the collector if the no collector has been specified
# from the command line or via the test deck.
if not net_test_loader.requiredTestHelpers and \
net_test_loader in requires_collector:
log.msg("Using the default collector: %s" %
response['default']['collector'])
net_test_loader.collector = response['default']['collector'].encode('utf-8')
continue
for th in net_test_loader.requiredTestHelpers:
# Only set helpers which are not already specified
if th['name'] not in required_test_helpers:
continue
test_helper = response[th['name']]
log.msg("Using this helper: %s" % test_helper)
th['test_class'].localOptions[th['option']] = test_helper['address'].encode('utf-8')
if net_test_loader in requires_collector:
net_test_loader.collector = test_helper['collector'].encode('utf-8')
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
""" fetch and verify a single NetTest's inputs """
log.debug("Fetching and verifying inputs")
for i in net_test_loader.inputFiles:
if 'url' in i:
log.debug("Downloading %s" % i['url'])
self.oonibclient.address = i['address']
try:
input_file = yield self.oonibclient.downloadInput(i['hash'])
except:
raise e.UnableToLoadDeckInput
try:
input_file.verify()
except AssertionError:
raise e.UnableToLoadDeckInput
i['test_class'].localOptions[i['key']] = input_file.cached_file
class Deck(InputFile):
def __init__(self, deck_hash=None,
deckFile=None,
decks_directory=config.decks_directory,
no_collector=False):
self.id = deck_hash
self.requiresTor = False
self.no_collector = no_collector
self.bouncer = ''
self.netTestLoaders = []
self.inputs = []
self.oonibclient = OONIBClient(self.bouncer)
self.decksDirectory = os.path.abspath(decks_directory)
self.deckHash = deck_hash
if deckFile:
self.loadDeck(deckFile)
@property
def cached_file(self):
return os.path.join(self.decksDirectory, self.deckHash)
@property
def cached_descriptor(self):
return self.cached_file + '.desc'
def loadDeck(self, deckFile):
with open(deckFile) as f:
self.deckHash = sha256(f.read()).hexdigest()
f.seek(0)
test_deck = yaml.safe_load(f)
for test in test_deck:
try:
nettest_path = nettest_to_path(test['options']['test_file'])
except e.NetTestNotFound:
log.err("Could not find %s" % test['options']['test_file'])
log.msg("Skipping...")
continue
net_test_loader = NetTestLoader(test['options']['subargs'],
annotations=test['options'].get('annotations', {}),
test_file=nettest_path)
if test['options']['collector']:
net_test_loader.collector = test['options']['collector']
self.insert(net_test_loader)
def insert(self, net_test_loader):
""" Add a NetTestLoader to this test deck """
try:
net_test_loader.checkOptions()
if net_test_loader.requiresTor:
self.requiresTor = True
except e.MissingTestHelper:
if not self.bouncer:
raise
self.requiresTor = True
if net_test_loader.collector and net_test_loader.collector.startswith('https://'):
_twisted_14_0_2_version = Version('twisted', 14, 0, 2)
if _twisted_version < _twisted_14_0_2_version:
raise e.HTTPCollectorUnsupported
elif net_test_loader.collector and net_test_loader.collector.startswith('http://'):
if config.advanced.insecure_collector is not True:
raise e.InsecureCollector
self.netTestLoaders.append(net_test_loader)
@defer.inlineCallbacks
def setup(self):
""" fetch and verify inputs for all NetTests in the deck """
log.msg("Fetching required net test inputs...")
for net_test_loader in self.netTestLoaders:
yield self.fetchAndVerifyNetTestInput(net_test_loader)
if self.bouncer:
log.msg("Looking up collector and test helpers")
yield self.lookupCollector()
@defer.inlineCallbacks
def lookupCollector(self):
self.oonibclient.address = self.bouncer
required_nettests = []
requires_test_helpers = False
requires_collector = False
for net_test_loader in self.netTestLoaders:
nettest = {
'name': net_test_loader.testName,
'version': net_test_loader.testVersion,
'test-helpers': [],
'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
}
if not net_test_loader.collector and not self.no_collector:
requires_collector = True
if len(net_test_loader.missingTestHelpers) > 0:
requires_test_helpers = True
nettest['test-helpers'] += map(lambda x: x[1],
net_test_loader.missingTestHelpers)
required_nettests.append(nettest)
if not requires_test_helpers and not requires_collector:
defer.returnValue(None)
log.debug("Looking up {}".format(required_nettests))
response = yield self.oonibclient.lookupTestCollector(required_nettests)
provided_net_tests = response['net-tests']
def find_collector_and_test_helpers(test_name, test_version, input_files):
input_files = [u""+x['hash'] for x in input_files]
for net_test in provided_net_tests:
if net_test['name'] != test_name:
continue
if net_test['version'] != test_version:
continue
if set(net_test['input-hashes']) != set(input_files):
continue
return net_test['collector'], net_test['test-helpers']
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" %
net_test_loader.testName)
collector, test_helpers = \
find_collector_and_test_helpers(net_test_loader.testName,
net_test_loader.testVersion,
net_test_loader.inputFiles)
for option, name in net_test_loader.missingTestHelpers:
test_helper_address = test_helpers[name].encode('utf-8')
net_test_loader.localOptions[option] = test_helper_address
net_test_loader.testHelpers[option] = test_helper_address
if not net_test_loader.collector:
net_test_loader.collector = collector.encode('utf-8')
@defer.inlineCallbacks
def lookupTestHelpers(self):
self.oonibclient.address = self.bouncer
required_test_helpers = []
requires_collector = []
for net_test_loader in self.netTestLoaders:
if not net_test_loader.collector and not self.no_collector:
requires_collector.append(net_test_loader)
required_test_helpers += map(lambda x: x[1],
net_test_loader.missingTestHelpers)
if not required_test_helpers and not requires_collector:
defer.returnValue(None)
response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" %
net_test_loader.testName)
# Only set the collector if the no collector has been specified
# from the command line or via the test deck.
if len(net_test_loader.missingTestHelpers) == 0 and \
net_test_loader in requires_collector:
log.msg("Using the default collector: %s" %
response['default']['collector'])
net_test_loader.collector = response['default']['collector'].encode('utf-8')
continue
for option, name in net_test_loader.missingTestHelpers:
test_helper_address = response[name]['address'].encode('utf-8')
test_helper_collector = \
response[name]['collector'].encode('utf-8')
log.msg("Using this helper: %s" % test_helper_address)
net_test_loader.localOptions[option] = test_helper_address
net_test_loader.testHelpers[option] = test_helper_address
if net_test_loader in requires_collector:
net_test_loader.collector = test_helper_collector
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
""" fetch and verify a single NetTest's inputs """
log.debug("Fetching and verifying inputs")
for i in net_test_loader.inputFiles:
if i['url']:
log.debug("Downloading %s" % i['url'])
self.oonibclient.address = i['address']
try:
input_file = yield self.oonibclient.downloadInput(i['hash'])
except:
raise e.UnableToLoadDeckInput
try:
input_file.verify()
except AssertionError:
raise e.UnableToLoadDeckInput
i['test_options'][i['key']] = input_file.cached_file
class Deck(InputFile):
def __init__(self, deck_hash=None,
deckFile=None,
decks_directory=config.decks_directory,
no_collector=False):
self.id = deck_hash
self.requiresTor = False
self.no_collector = no_collector
self.bouncer = ''
self.netTestLoaders = []
self.inputs = []
self.oonibclient = OONIBClient(self.bouncer)
self.decksDirectory = os.path.abspath(decks_directory)
self.deckHash = deck_hash
if deckFile:
self.loadDeck(deckFile)
@property
def cached_file(self):
return os.path.join(self.decksDirectory, self.deckHash)
@property
def cached_descriptor(self):
return self.cached_file + '.desc'
def loadDeck(self, deckFile):
with open(deckFile) as f:
self.deckHash = sha256(f.read()).hexdigest()
f.seek(0)
test_deck = yaml.safe_load(f)
for test in test_deck:
try:
nettest_path = nettest_to_path(test['options']['test_file'])
except e.NetTestNotFound:
log.err("Could not find %s" % test['options']['test_file'])
log.msg("Skipping...")
continue
net_test_loader = NetTestLoader(test['options']['subargs'],
test_file=nettest_path)
if test['options']['collector']:
net_test_loader.collector = test['options']['collector']
self.insert(net_test_loader)
def insert(self, net_test_loader):
""" Add a NetTestLoader to this test deck """
def has_test_helper(missing_option):
for rth in net_test_loader.requiredTestHelpers:
if missing_option == rth['option']:
return True
return False
try:
net_test_loader.checkOptions()
if net_test_loader.requiresTor:
self.requiresTor = True
except e.MissingRequiredOption as missing_options:
if not self.bouncer:
raise
for missing_option in missing_options.message:
if not has_test_helper(missing_option):
raise
self.requiresTor = True
if net_test_loader.collector and net_test_loader.collector.startswith('https://'):
_twisted_14_0_2_version = Version('twisted', 14, 0, 2)
if _twisted_version < _twisted_14_0_2_version:
raise e.HTTPCollectorUnsupported
elif net_test_loader.collector and net_test_loader.collector.startswith('http://'):
if config.advanced.insecure_collector is not True:
raise e.InsecureCollector
self.netTestLoaders.append(net_test_loader)
@defer.inlineCallbacks
def setup(self):
""" fetch and verify inputs for all NetTests in the deck """
log.msg("Fetching required net test inputs...")
for net_test_loader in self.netTestLoaders:
yield self.fetchAndVerifyNetTestInput(net_test_loader)
if self.bouncer:
log.msg("Looking up collector and test helpers")
yield self.lookupCollector()
@defer.inlineCallbacks
def lookupCollector(self):
self.oonibclient.address = self.bouncer
required_nettests = []
requires_test_helpers = False
requires_collector = False
for net_test_loader in self.netTestLoaders:
nettest = {
'name': net_test_loader.testDetails['test_name'],
'version': net_test_loader.testDetails['test_version'],
'test-helpers': [],
'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
}
if not net_test_loader.collector and not self.no_collector:
requires_collector = True
for th in net_test_loader.requiredTestHelpers:
# {'name':'', 'option':'', 'test_class':''}
if th['test_class'].localOptions[th['option']]:
continue
nettest['test-helpers'].append(th['name'])
requires_test_helpers = True
required_nettests.append(nettest)
if not requires_test_helpers and not requires_collector:
defer.returnValue(None)
response = yield self.oonibclient.lookupTestCollector(required_nettests)
provided_net_tests = response['net-tests']
def find_collector_and_test_helpers(test_name, test_version, input_files):
input_files = [u""+x['hash'] for x in input_files]
for net_test in provided_net_tests:
if net_test['name'] != test_name:
continue
if net_test['version'] != test_version:
continue
if set(net_test['input-hashes']) != set(input_files):
continue
return net_test['collector'], net_test['test-helpers']
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])
collector, test_helpers = \
find_collector_and_test_helpers(net_test_loader.testDetails['test_name'],
net_test_loader.testDetails['test_version'],
net_test_loader.inputFiles)
for th in net_test_loader.requiredTestHelpers:
if not th['test_class'].localOptions[th['option']]:
th['test_class'].localOptions[th['option']] = test_helpers[th['name']].encode('utf-8')
net_test_loader.testHelpers[th['option']] = th['test_class'].localOptions[th['option']]
if not net_test_loader.collector:
net_test_loader.collector = collector.encode('utf-8')
@defer.inlineCallbacks
def lookupTestHelpers(self):
self.oonibclient.address = self.bouncer
required_test_helpers = []
requires_collector = []
for net_test_loader in self.netTestLoaders:
if not net_test_loader.collector and not self.no_collector:
requires_collector.append(net_test_loader)
for th in net_test_loader.requiredTestHelpers:
# {'name':'', 'option':'', 'test_class':''}
if th['test_class'].localOptions[th['option']]:
continue
required_test_helpers.append(th['name'])
if not required_test_helpers and not requires_collector:
defer.returnValue(None)
response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)
for net_test_loader in self.netTestLoaders:
log.msg("Setting collector and test helpers for %s" %
net_test_loader.testDetails['test_name'])
# Only set the collector if the no collector has been specified
# from the command line or via the test deck.
if not net_test_loader.requiredTestHelpers and \
net_test_loader in requires_collector:
log.msg("Using the default collector: %s" %
response['default']['collector'])
net_test_loader.collector = response['default']['collector'].encode('utf-8')
continue
for th in net_test_loader.requiredTestHelpers:
# Only set helpers which are not already specified
if th['name'] not in required_test_helpers:
continue
test_helper = response[th['name']]
log.msg("Using this helper: %s" % test_helper)
th['test_class'].localOptions[th['option']] = test_helper['address'].encode('utf-8')
if net_test_loader in requires_collector:
net_test_loader.collector = test_helper['collector'].encode('utf-8')
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
""" fetch and verify a single NetTest's inputs """
log.debug("Fetching and verifying inputs")
for i in net_test_loader.inputFiles:
if 'url' in i:
log.debug("Downloading %s" % i['url'])
self.oonibclient.address = i['address']
try:
input_file = yield self.oonibclient.downloadInput(i['hash'])
except:
raise e.UnableToLoadDeckInput
try:
input_file.verify()
except AssertionError:
raise e.UnableToLoadDeckInput
i['test_class'].localOptions[i['key']] = input_file.cached_file
