Esempio n. 1
0
    def lookupTestHelpers(self):
        from ooni.oonibclient import OONIBClient
        oonibclient = OONIBClient(self.bouncer)
        required_test_helpers = []
        requires_collector = []
        for net_test_loader in self.netTestLoaders:
            if not net_test_loader.collector:
                requires_collector.append(net_test_loader)

            for th in net_test_loader.requiredTestHelpers:
                # {'name':'', 'option':'', 'test_class':''}
                if th['test_class'].localOptions[th['option']]:
                    continue
                required_test_helpers.append(th['name'])
        
        if not required_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield oonibclient.lookupTestHelpers(required_test_helpers)

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])

            # Only set the collector if the no collector has been specified
            # from the command line or via the test deck.
            if not net_test_loader.requiredTestHelpers and net_test_loader in requires_collector:
                log.msg("Using the default collector: %s" % response['default']['collector'])
                net_test_loader.collector = response['default']['collector'].encode('utf-8')

            for th in net_test_loader.requiredTestHelpers:
                test_helper = response[th['name']]
                log.msg("Using this helper: %s" % test_helper)
                th['test_class'].localOptions[th['option']] = test_helper['address']
                if net_test_loader in requires_collector:
                    net_test_loader.collector = test_helper['collector'].encode('utf-8')
Esempio n. 2
0
def upload(report_file, collector=None, bouncer=None):
    print "Attempting to upload %s" % report_file

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer:
        oonib_client = OONIBClient(bouncer)
        collector = yield oonib_client.lookupTestCollector(report.header["test_name"])

    if collector is None:
        try:
            collector = report_log[report_file]["collector"]
            if collector is None:
                raise KeyError
        except KeyError:
            raise Exception("No collector or bouncer specified and collector not in report log.")

    oonib_reporter = OONIBReporter(report.header, collector)
    log.msg("Creating report for %s with %s" % (report_file, collector))
    report_id = yield oonib_reporter.createReport()
    yield oonib_report_log.created(report_file, collector, report_id)
    for entry in report:
        print "Writing entry"
        yield oonib_reporter.writeReportEntry(entry)
    log.msg("Closing report.")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Esempio n. 3
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and not collector:
        oonib_client = OONIBClient(bouncer)
        net_tests = [{
            'test-helpers': [],
            'input-hashes': report.header['input_hashes'],
            'name': report.header['test_name'],
            'version': report.header['test_version'],
        }]
        result = yield oonib_client.lookupTestCollector(
            net_tests
        )
        collector = str(result['net-tests'][0]['collector'])

    if collector is None:
        try:
            collector = report_log[report_file]['collector']
            if collector is None:
                raise KeyError
        except KeyError:
            raise Exception(
                "No collector or bouncer specified"
                " and collector not in report log."
            )

    oonib_reporter = OONIBReporter(report.header, collector)
    log.msg("Creating report for %s with %s" % (report_file, collector))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file, collector, report_id)
    log.msg("Writing report entries")
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
        sys.stdout.write('.')
        sys.stdout.flush()
    log.msg("Closing report")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Esempio n. 4
0
    def __init__(self, deck_hash=None,
                 deckFile=None,
                 decks_directory=config.decks_directory,
                 no_collector=False):
        self.id = deck_hash
        self.requiresTor = False
        self.no_collector = no_collector
        self.bouncer = ''
        self.netTestLoaders = []
        self.inputs = []

        self.oonibclient = OONIBClient(self.bouncer)

        self.decksDirectory = os.path.abspath(decks_directory)
        self.deckHash = deck_hash

        if deckFile:
            self.loadDeck(deckFile)
Esempio n. 5
0
def upload(report_file, collector=None, bouncer=None):
    oonib_report_log = OONIBReportLog()

    log.msg("Attempting to upload %s" % report_file)

    with open(config.report_log_file) as f:
        report_log = yaml.safe_load(f)

    report = parser.ReportLoader(report_file)
    if bouncer and not collector:
        oonib_client = OONIBClient(bouncer)
        net_tests = [{
            'test-helpers': [],
            'input-hashes': report.header['input_hashes'],
            'name': report.header['test_name'],
            'version': report.header['test_version'],
        }]
        result = yield oonib_client.lookupTestCollector(
            net_tests
        )
        collector = str(result['net-tests'][0]['collector'])

    if collector is None:
        try:
            collector = report_log[report_file]['collector']
            if collector is None:
                raise KeyError
        except KeyError:
            raise Exception(
                "No collector or bouncer specified"
                " and collector not in report log."
            )

    oonib_reporter = OONIBReporter(report.header, collector)
    log.msg("Creating report for %s with %s" % (report_file, collector))
    report_id = yield oonib_reporter.createReport()
    report.header['report_id'] = report_id
    yield oonib_report_log.created(report_file, collector, report_id)
    for entry in report:
        yield oonib_reporter.writeReportEntry(entry)
    log.msg("Closing report.")
    yield oonib_reporter.finish()
    yield oonib_report_log.closed(report_file)
Esempio n. 6
0
    def fetchAndVerifyNetTestInput(self, net_test_loader):
        """ fetch and verify a single NetTest's inputs """
        from ooni.oonibclient import OONIBClient
        log.debug("Fetching and verifying inputs")
        for i in net_test_loader.inputFiles:
            if 'url' in i:
                log.debug("Downloading %s" % i['url'])
                oonibclient = OONIBClient(i['address'])
                
                try:
                    input_file = yield oonibclient.downloadInput(i['hash'])
                except:
                    raise e.UnableToLoadDeckInput

                try:
                    input_file.verify()
                except AssertionError:
                    raise e.UnableToLoadDeckInput, cached_path
                
                i['test_class'].localOptions[i['key']] = input_file.cached_file
Esempio n. 7
0
    def setUp(self):
        super(TestOONIBClient, self).setUp()
        host = '127.0.0.1'
        port = 8889
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host, port))
            s.shutdown(2)

            data_dir = '/tmp/testooni'
            config.advanced.data_dir = data_dir

            if os.path.exists(data_dir):
                shutil.rmtree(data_dir)
            os.mkdir(data_dir)
            os.mkdir(os.path.join(data_dir, 'inputs'))
            os.mkdir(os.path.join(data_dir, 'decks'))
        except Exception:
            self.skipTest(
                "OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)"
            )
        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
Esempio n. 8
0
 def setUp(self):
     host = '127.0.0.1'
     port = 8889
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
         s.connect((host, port))
         s.shutdown(2)
         try: shutil.rmtree(data_dir)
         except: pass
         os.mkdir(data_dir)
         os.mkdir(os.path.join(data_dir, 'inputs'))
         os.mkdir(os.path.join(data_dir, 'decks'))
     except Exception as ex:
         self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
     self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
Esempio n. 9
0
    def __init__(self, deck_hash=None,
                 deckFile=None,
                 decks_directory=config.decks_directory):
        self.id = deck_hash
        self.requiresTor = False
        self.bouncer = ''
        self.netTestLoaders = []
        self.inputs = []
        self.testHelpers = {}

        self.oonibclient = OONIBClient(self.bouncer)

        self.decksDirectory = os.path.abspath(decks_directory)
        self.deckHash = deck_hash

        if deckFile: self.loadDeck(deckFile)
Esempio n. 10
0
    def setUp(self):
        super(TestOONIBClient, self).setUp()
        host = '127.0.0.1'
        port = 8889
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host, port))
            s.shutdown(2)

            data_dir = '/tmp/testooni'
            config.advanced.data_dir = data_dir

            if os.path.exists(data_dir):
                shutil.rmtree(data_dir)
            os.mkdir(data_dir)
            os.mkdir(os.path.join(data_dir, 'inputs'))
            os.mkdir(os.path.join(data_dir, 'decks'))
        except Exception:
            self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
Esempio n. 11
0
class Deck(InputFile):
    def __init__(self, deck_hash=None,
                 deckFile=None,
                 decks_directory=config.decks_directory,
                 no_collector=False):
        self.id = deck_hash
        self.requiresTor = False
        self.no_collector = no_collector
        self.bouncer = ''
        self.netTestLoaders = []
        self.inputs = []

        self.oonibclient = OONIBClient(self.bouncer)

        self.decksDirectory = os.path.abspath(decks_directory)
        self.deckHash = deck_hash

        if deckFile:
            self.loadDeck(deckFile)

    @property
    def cached_file(self):
        return os.path.join(self.decksDirectory, self.deckHash)

    @property
    def cached_descriptor(self):
        return self.cached_file + '.desc'

    def loadDeck(self, deckFile):
        with open(deckFile) as f:
            self.deckHash = sha256(f.read()).hexdigest()
            f.seek(0)
            test_deck = yaml.safe_load(f)

        for test in test_deck:
            try:
                nettest_path = nettest_to_path(test['options']['test_file'])
            except e.NetTestNotFound:
                log.err("Could not find %s" % test['options']['test_file'])
                log.msg("Skipping...")
                continue
            net_test_loader = NetTestLoader(test['options']['subargs'],
                                            test_file=nettest_path)
            if test['options']['collector']:
                net_test_loader.collector = test['options']['collector']
            self.insert(net_test_loader)

    def insert(self, net_test_loader):
        """ Add a NetTestLoader to this test deck """

        def has_test_helper(missing_option):
            for rth in net_test_loader.requiredTestHelpers:
                if missing_option == rth['option']:
                    return True
            return False

        try:
            net_test_loader.checkOptions()
            if net_test_loader.requiresTor:
                self.requiresTor = True
        except e.MissingRequiredOption as missing_options:
            if not self.bouncer:
                raise
            for missing_option in missing_options.message:
                if not has_test_helper(missing_option):
                    raise
            self.requiresTor = True
        self.netTestLoaders.append(net_test_loader)

    @defer.inlineCallbacks
    def setup(self):
        """ fetch and verify inputs for all NetTests in the deck """
        log.msg("Fetching required net test inputs...")
        for net_test_loader in self.netTestLoaders:
            yield self.fetchAndVerifyNetTestInput(net_test_loader)

        if self.bouncer:
            log.msg("Looking up collector and test helpers")
            yield self.lookupCollector()

    @defer.inlineCallbacks
    def lookupCollector(self):
        self.oonibclient.address = self.bouncer

        required_nettests = []

        requires_test_helpers = False
        requires_collector = False
        for net_test_loader in self.netTestLoaders:
            nettest = {
                'name': net_test_loader.testDetails['test_name'],
                'version': net_test_loader.testDetails['test_version'],
                'test-helpers': [],
                'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
            }
            if not net_test_loader.collector and not self.no_collector:
                requires_collector = True

            for th in net_test_loader.requiredTestHelpers:
                # {'name':'', 'option':'', 'test_class':''}
                if th['test_class'].localOptions[th['option']]:
                    continue
                nettest['test-helpers'].append(th['name'])
                requires_test_helpers = True

            required_nettests.append(nettest)

        if not requires_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield self.oonibclient.lookupTestCollector(required_nettests)
        provided_net_tests = response['net-tests']

        def find_collector_and_test_helpers(test_name, test_version, input_files):
            input_files = [u""+x['hash'] for x in input_files]
            for net_test in provided_net_tests:
                if net_test['name'] != test_name:
                    continue
                if net_test['version'] != test_version:
                    continue
                if set(net_test['input-hashes']) != set(input_files):
                    continue
                return net_test['collector'], net_test['test-helpers']

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])

            collector, test_helpers = \
                find_collector_and_test_helpers(net_test_loader.testDetails['test_name'],
                                                net_test_loader.testDetails['test_version'],
                                                net_test_loader.inputFiles)

            for th in net_test_loader.requiredTestHelpers:
                if not th['test_class'].localOptions[th['option']]:
                    th['test_class'].localOptions[th['option']] = test_helpers[th['name']].encode('utf-8')
                net_test_loader.testHelpers[th['option']] = th['test_class'].localOptions[th['option']]

            if not net_test_loader.collector:
                net_test_loader.collector = collector.encode('utf-8')

    @defer.inlineCallbacks
    def lookupTestHelpers(self):
        self.oonibclient.address = self.bouncer

        required_test_helpers = []
        requires_collector = []
        for net_test_loader in self.netTestLoaders:
            if not net_test_loader.collector and not self.no_collector:
                requires_collector.append(net_test_loader)

            for th in net_test_loader.requiredTestHelpers:
                # {'name':'', 'option':'', 'test_class':''}
                if th['test_class'].localOptions[th['option']]:
                    continue
                required_test_helpers.append(th['name'])

        if not required_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" %
                    net_test_loader.testDetails['test_name'])

            # Only set the collector if the no collector has been specified
            # from the command line or via the test deck.
            if not net_test_loader.requiredTestHelpers and \
                            net_test_loader in requires_collector:
                log.msg("Using the default collector: %s" %
                        response['default']['collector'])
                net_test_loader.collector = response['default']['collector'].encode('utf-8')
                continue

            for th in net_test_loader.requiredTestHelpers:
                # Only set helpers which are not already specified
                if th['name'] not in required_test_helpers:
                    continue
                test_helper = response[th['name']]
                log.msg("Using this helper: %s" % test_helper)
                th['test_class'].localOptions[th['option']] = test_helper['address'].encode('utf-8')
                if net_test_loader in requires_collector:
                    net_test_loader.collector = test_helper['collector'].encode('utf-8')

    @defer.inlineCallbacks
    def fetchAndVerifyNetTestInput(self, net_test_loader):
        """ fetch and verify a single NetTest's inputs """
        log.debug("Fetching and verifying inputs")
        for i in net_test_loader.inputFiles:
            if 'url' in i:
                log.debug("Downloading %s" % i['url'])
                self.oonibclient.address = i['address']

                try:
                    input_file = yield self.oonibclient.downloadInput(i['hash'])
                except:
                    raise e.UnableToLoadDeckInput

                try:
                    input_file.verify()
                except AssertionError:
                    raise e.UnableToLoadDeckInput

                i['test_class'].localOptions[i['key']] = input_file.cached_file
class TestOONIBClient(ConfigTestCase):
    def setUp(self):
        host = '127.0.0.1'
        port = 8889
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host, port))
            s.shutdown(2)

            data_dir = '/tmp/testooni'
            config.advanced.data_dir = data_dir

            try:
                shutil.rmtree(data_dir)
            except:
                pass
            os.mkdir(data_dir)
            os.mkdir(os.path.join(data_dir, 'inputs'))
            os.mkdir(os.path.join(data_dir, 'decks'))
        except Exception as ex:
            self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))

    @defer.inlineCallbacks
    def test_query(self):
        res = yield self.oonibclient.queryBackend('GET', '/policy/input')
        self.assertTrue(isinstance(res, list))

    @defer.inlineCallbacks
    def test_get_input_list(self):
        input_list = yield self.oonibclient.getInputList()
        self.assertTrue(isinstance(input_list, list))

    @defer.inlineCallbacks
    def test_get_input_descriptor(self):
        input_descriptor = yield self.oonibclient.getInput(input_id)
        for key in ['name', 'description',
                    'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(input_descriptor, key))

    @defer.inlineCallbacks
    def test_download_input(self):
        yield self.oonibclient.downloadInput(input_id)

    @defer.inlineCallbacks
    def test_get_deck_list(self):
        deck_list = yield self.oonibclient.getDeckList()
        self.assertTrue(isinstance(deck_list, list))

    @defer.inlineCallbacks
    def test_get_deck_descriptor(self):
        deck_descriptor = yield self.oonibclient.getDeck(deck_id)
        for key in ['name', 'description',
                    'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(deck_descriptor, key))

    @defer.inlineCallbacks
    def test_download_deck(self):
        yield self.oonibclient.downloadDeck(deck_id)

    def test_lookup_invalid_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        return self.failUnlessFailure(
            self.oonibclient.lookupTestHelpers([
                'sdadsadsa', 'dns'
            ]), e.CouldNotFindTestHelper)

    @defer.inlineCallbacks
    def test_lookup_no_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = []
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertTrue('default' in helpers.keys())

    @defer.inlineCallbacks
    def test_lookup_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = [u'http-return-json-headers', u'dns']
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))
        self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http'))
        self.assertTrue(int(helpers['dns']['address'].split('.')[0]))

    @defer.inlineCallbacks
    def test_input_descriptor_not_found(self):
        try:
            yield self.oonibclient.queryBackend('GET', '/input/' + 'a'*64)
        except e.OONIBInputDescriptorNotFound:
            pass
        else:
            assert False

    @defer.inlineCallbacks
    def test_http_errors(self):
        try:
            yield self.oonibclient.queryBackend('PUT', '/policy/input')
        except error.Error:
            pass
        else:
            assert False

    @defer.inlineCallbacks
    def test_create_report(self):
        res = yield self.oonibclient.queryBackend('POST', '/report', {
            'software_name': 'spam',
            'software_version': '2.0',
            'probe_asn': 'AS0',
            'probe_cc': 'ZZ',
            'test_name': 'foobar',
            'test_version': '1.0',
            'input_hashes': []
        })
        assert isinstance(res['report_id'], unicode)

    @defer.inlineCallbacks
    def test_report_lifecycle(self):
        res = yield self.oonibclient.queryBackend('POST', '/report', {
            'software_name': 'spam',
            'software_version': '2.0',
            'probe_asn': 'AS0',
            'probe_cc': 'ZZ',
            'test_name': 'foobar',
            'test_version': '1.0',
            'input_hashes': []
        })
        report_id = str(res['report_id'])

        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, {
            'content': '---\nspam: ham\n...\n'
        })

        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, {
            'content': '---\nspam: ham\n...\n'
        })

        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id + '/close')
Esempio n. 13
0
class Deck(InputFile):
    def __init__(self, deck_hash=None,
                 deckFile=None,
                 decks_directory=config.decks_directory,
                 no_collector=False):
        self.id = deck_hash
        self.requiresTor = False
        self.no_collector = no_collector
        self.bouncer = ''
        self.netTestLoaders = []
        self.inputs = []

        self.oonibclient = OONIBClient(self.bouncer)

        self.decksDirectory = os.path.abspath(decks_directory)
        self.deckHash = deck_hash

        if deckFile:
            self.loadDeck(deckFile)

    @property
    def cached_file(self):
        return os.path.join(self.decksDirectory, self.deckHash)

    @property
    def cached_descriptor(self):
        return self.cached_file + '.desc'

    def loadDeck(self, deckFile):
        with open(deckFile) as f:
            self.deckHash = sha256(f.read()).hexdigest()
            f.seek(0)
            test_deck = yaml.safe_load(f)

        for test in test_deck:
            try:
                nettest_path = nettest_to_path(test['options']['test_file'])
            except e.NetTestNotFound:
                log.err("Could not find %s" % test['options']['test_file'])
                log.msg("Skipping...")
                continue
            net_test_loader = NetTestLoader(test['options']['subargs'],
                                            annotations=test['options'].get('annotations', {}),
                                            test_file=nettest_path)
            if test['options']['collector']:
                net_test_loader.collector = test['options']['collector']
            self.insert(net_test_loader)

    def insert(self, net_test_loader):
        """ Add a NetTestLoader to this test deck """
        try:
            net_test_loader.checkOptions()
            if net_test_loader.requiresTor:
                self.requiresTor = True
        except e.MissingTestHelper:
            if not self.bouncer:
                raise
            self.requiresTor = True

        if net_test_loader.collector and net_test_loader.collector.startswith('https://'):
            _twisted_14_0_2_version = Version('twisted', 14, 0, 2)
            if _twisted_version < _twisted_14_0_2_version:
                raise e.HTTPCollectorUnsupported
        elif net_test_loader.collector and net_test_loader.collector.startswith('http://'):
            if config.advanced.insecure_collector is not True:
                raise e.InsecureCollector
        self.netTestLoaders.append(net_test_loader)

    @defer.inlineCallbacks
    def setup(self):
        """ fetch and verify inputs for all NetTests in the deck """
        log.msg("Fetching required net test inputs...")
        for net_test_loader in self.netTestLoaders:
            yield self.fetchAndVerifyNetTestInput(net_test_loader)

        if self.bouncer:
            log.msg("Looking up collector and test helpers")
            yield self.lookupCollector()

    @defer.inlineCallbacks
    def lookupCollector(self):
        self.oonibclient.address = self.bouncer

        required_nettests = []

        requires_test_helpers = False
        requires_collector = False
        for net_test_loader in self.netTestLoaders:
            nettest = {
                'name': net_test_loader.testName,
                'version': net_test_loader.testVersion,
                'test-helpers': [],
                'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
            }
            if not net_test_loader.collector and not self.no_collector:
                requires_collector = True

            if len(net_test_loader.missingTestHelpers) > 0:
                requires_test_helpers = True
                nettest['test-helpers'] += map(lambda x: x[1],
                                               net_test_loader.missingTestHelpers)

            required_nettests.append(nettest)

        if not requires_test_helpers and not requires_collector:
            defer.returnValue(None)

        log.debug("Looking up {}".format(required_nettests))
        response = yield self.oonibclient.lookupTestCollector(required_nettests)
        provided_net_tests = response['net-tests']

        def find_collector_and_test_helpers(test_name, test_version, input_files):
            input_files = [u""+x['hash'] for x in input_files]
            for net_test in provided_net_tests:
                if net_test['name'] != test_name:
                    continue
                if net_test['version'] != test_version:
                    continue
                if set(net_test['input-hashes']) != set(input_files):
                    continue
                return net_test['collector'], net_test['test-helpers']

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" %
                    net_test_loader.testName)

            collector, test_helpers = \
                find_collector_and_test_helpers(net_test_loader.testName,
                                                net_test_loader.testVersion,
                                                net_test_loader.inputFiles)

            for option, name in net_test_loader.missingTestHelpers:
                test_helper_address = test_helpers[name].encode('utf-8')
                net_test_loader.localOptions[option] = test_helper_address
                net_test_loader.testHelpers[option] = test_helper_address

            if not net_test_loader.collector:
                net_test_loader.collector = collector.encode('utf-8')

    @defer.inlineCallbacks
    def lookupTestHelpers(self):
        self.oonibclient.address = self.bouncer

        required_test_helpers = []
        requires_collector = []
        for net_test_loader in self.netTestLoaders:
            if not net_test_loader.collector and not self.no_collector:
                requires_collector.append(net_test_loader)

            required_test_helpers += map(lambda x: x[1],
                                           net_test_loader.missingTestHelpers)

        if not required_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" %
                    net_test_loader.testName)

            # Only set the collector if the no collector has been specified
            # from the command line or via the test deck.
            if len(net_test_loader.missingTestHelpers) == 0 and \
                            net_test_loader in requires_collector:
                log.msg("Using the default collector: %s" %
                        response['default']['collector'])
                net_test_loader.collector = response['default']['collector'].encode('utf-8')
                continue

            for option, name in net_test_loader.missingTestHelpers:
                test_helper_address = response[name]['address'].encode('utf-8')
                test_helper_collector = \
                    response[name]['collector'].encode('utf-8')

                log.msg("Using this helper: %s" % test_helper_address)
                net_test_loader.localOptions[option] = test_helper_address
                net_test_loader.testHelpers[option] = test_helper_address
                if net_test_loader in requires_collector:
                    net_test_loader.collector = test_helper_collector

    @defer.inlineCallbacks
    def fetchAndVerifyNetTestInput(self, net_test_loader):
        """ fetch and verify a single NetTest's inputs """
        log.debug("Fetching and verifying inputs")
        for i in net_test_loader.inputFiles:
            if i['url']:
                log.debug("Downloading %s" % i['url'])
                self.oonibclient.address = i['address']

                try:
                    input_file = yield self.oonibclient.downloadInput(i['hash'])
                except:
                    raise e.UnableToLoadDeckInput

                try:
                    input_file.verify()
                except AssertionError:
                    raise e.UnableToLoadDeckInput

                i['test_options'][i['key']] = input_file.cached_file
Esempio n. 14
0
class Deck(InputFile):
    def __init__(self, deck_hash=None,
                 deckFile=None,
                 decks_directory=config.decks_directory,
                 no_collector=False):
        self.id = deck_hash
        self.requiresTor = False
        self.no_collector = no_collector
        self.bouncer = ''
        self.netTestLoaders = []
        self.inputs = []

        self.oonibclient = OONIBClient(self.bouncer)

        self.decksDirectory = os.path.abspath(decks_directory)
        self.deckHash = deck_hash

        if deckFile:
            self.loadDeck(deckFile)

    @property
    def cached_file(self):
        return os.path.join(self.decksDirectory, self.deckHash)

    @property
    def cached_descriptor(self):
        return self.cached_file + '.desc'

    def loadDeck(self, deckFile):
        with open(deckFile) as f:
            self.deckHash = sha256(f.read()).hexdigest()
            f.seek(0)
            test_deck = yaml.safe_load(f)

        for test in test_deck:
            try:
                nettest_path = nettest_to_path(test['options']['test_file'])
            except e.NetTestNotFound:
                log.err("Could not find %s" % test['options']['test_file'])
                log.msg("Skipping...")
                continue
            net_test_loader = NetTestLoader(test['options']['subargs'],
                                            test_file=nettest_path)
            if test['options']['collector']:
                net_test_loader.collector = test['options']['collector']
            self.insert(net_test_loader)

    def insert(self, net_test_loader):
        """ Add a NetTestLoader to this test deck """

        def has_test_helper(missing_option):
            for rth in net_test_loader.requiredTestHelpers:
                if missing_option == rth['option']:
                    return True
            return False

        try:
            net_test_loader.checkOptions()
            if net_test_loader.requiresTor:
                self.requiresTor = True
        except e.MissingRequiredOption as missing_options:
            if not self.bouncer:
                raise
            for missing_option in missing_options.message:
                if not has_test_helper(missing_option):
                    raise
            self.requiresTor = True

        if net_test_loader.collector and net_test_loader.collector.startswith('https://'):
            _twisted_14_0_2_version = Version('twisted', 14, 0, 2)
            if _twisted_version < _twisted_14_0_2_version:
                raise e.HTTPCollectorUnsupported
        elif net_test_loader.collector and net_test_loader.collector.startswith('http://'):
            if config.advanced.insecure_collector is not True:
                raise e.InsecureCollector
        self.netTestLoaders.append(net_test_loader)

    @defer.inlineCallbacks
    def setup(self):
        """ fetch and verify inputs for all NetTests in the deck """
        log.msg("Fetching required net test inputs...")
        for net_test_loader in self.netTestLoaders:
            yield self.fetchAndVerifyNetTestInput(net_test_loader)

        if self.bouncer:
            log.msg("Looking up collector and test helpers")
            yield self.lookupCollector()

    @defer.inlineCallbacks
    def lookupCollector(self):
        self.oonibclient.address = self.bouncer

        required_nettests = []

        requires_test_helpers = False
        requires_collector = False
        for net_test_loader in self.netTestLoaders:
            nettest = {
                'name': net_test_loader.testDetails['test_name'],
                'version': net_test_loader.testDetails['test_version'],
                'test-helpers': [],
                'input-hashes': [x['hash'] for x in net_test_loader.inputFiles]
            }
            if not net_test_loader.collector and not self.no_collector:
                requires_collector = True

            for th in net_test_loader.requiredTestHelpers:
                # {'name':'', 'option':'', 'test_class':''}
                if th['test_class'].localOptions[th['option']]:
                    continue
                nettest['test-helpers'].append(th['name'])
                requires_test_helpers = True

            required_nettests.append(nettest)

        if not requires_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield self.oonibclient.lookupTestCollector(required_nettests)
        provided_net_tests = response['net-tests']

        def find_collector_and_test_helpers(test_name, test_version, input_files):
            input_files = [u""+x['hash'] for x in input_files]
            for net_test in provided_net_tests:
                if net_test['name'] != test_name:
                    continue
                if net_test['version'] != test_version:
                    continue
                if set(net_test['input-hashes']) != set(input_files):
                    continue
                return net_test['collector'], net_test['test-helpers']

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])

            collector, test_helpers = \
                find_collector_and_test_helpers(net_test_loader.testDetails['test_name'],
                                                net_test_loader.testDetails['test_version'],
                                                net_test_loader.inputFiles)

            for th in net_test_loader.requiredTestHelpers:
                if not th['test_class'].localOptions[th['option']]:
                    th['test_class'].localOptions[th['option']] = test_helpers[th['name']].encode('utf-8')
                net_test_loader.testHelpers[th['option']] = th['test_class'].localOptions[th['option']]

            if not net_test_loader.collector:
                net_test_loader.collector = collector.encode('utf-8')

    @defer.inlineCallbacks
    def lookupTestHelpers(self):
        self.oonibclient.address = self.bouncer

        required_test_helpers = []
        requires_collector = []
        for net_test_loader in self.netTestLoaders:
            if not net_test_loader.collector and not self.no_collector:
                requires_collector.append(net_test_loader)

            for th in net_test_loader.requiredTestHelpers:
                # {'name':'', 'option':'', 'test_class':''}
                if th['test_class'].localOptions[th['option']]:
                    continue
                required_test_helpers.append(th['name'])

        if not required_test_helpers and not requires_collector:
            defer.returnValue(None)

        response = yield self.oonibclient.lookupTestHelpers(required_test_helpers)

        for net_test_loader in self.netTestLoaders:
            log.msg("Setting collector and test helpers for %s" %
                    net_test_loader.testDetails['test_name'])

            # Only set the collector if the no collector has been specified
            # from the command line or via the test deck.
            if not net_test_loader.requiredTestHelpers and \
                            net_test_loader in requires_collector:
                log.msg("Using the default collector: %s" %
                        response['default']['collector'])
                net_test_loader.collector = response['default']['collector'].encode('utf-8')
                continue

            for th in net_test_loader.requiredTestHelpers:
                # Only set helpers which are not already specified
                if th['name'] not in required_test_helpers:
                    continue
                test_helper = response[th['name']]
                log.msg("Using this helper: %s" % test_helper)
                th['test_class'].localOptions[th['option']] = test_helper['address'].encode('utf-8')
                if net_test_loader in requires_collector:
                    net_test_loader.collector = test_helper['collector'].encode('utf-8')

    @defer.inlineCallbacks
    def fetchAndVerifyNetTestInput(self, net_test_loader):
        """ fetch and verify a single NetTest's inputs """
        log.debug("Fetching and verifying inputs")
        for i in net_test_loader.inputFiles:
            if 'url' in i:
                log.debug("Downloading %s" % i['url'])
                self.oonibclient.address = i['address']

                try:
                    input_file = yield self.oonibclient.downloadInput(i['hash'])
                except:
                    raise e.UnableToLoadDeckInput

                try:
                    input_file.verify()
                except AssertionError:
                    raise e.UnableToLoadDeckInput

                i['test_class'].localOptions[i['key']] = input_file.cached_file
Esempio n. 15
0
class TestOONIBClient(ConfigTestCase):
    def setUp(self):
        super(TestOONIBClient, self).setUp()
        host = '127.0.0.1'
        port = 8889
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host, port))
            s.shutdown(2)

            data_dir = '/tmp/testooni'
            config.advanced.data_dir = data_dir

            if os.path.exists(data_dir):
                shutil.rmtree(data_dir)
            os.mkdir(data_dir)
            os.mkdir(os.path.join(data_dir, 'inputs'))
            os.mkdir(os.path.join(data_dir, 'decks'))
        except Exception:
            self.skipTest(
                "OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)"
            )
        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))

    @defer.inlineCallbacks
    def test_query(self):
        res = yield self.oonibclient.queryBackend('GET', '/policy/input')
        self.assertTrue(isinstance(res, list))

    @defer.inlineCallbacks
    def test_get_input_list(self):
        input_list = yield self.oonibclient.getInputList()
        self.assertTrue(isinstance(input_list, list))

    @defer.inlineCallbacks
    def test_get_input_descriptor(self):
        input_descriptor = yield self.oonibclient.getInput(input_id)
        for key in ['name', 'description', 'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(input_descriptor, key))

    @defer.inlineCallbacks
    def test_download_input(self):
        yield self.oonibclient.downloadInput(input_id)

    @defer.inlineCallbacks
    def test_get_deck_list(self):
        deck_list = yield self.oonibclient.getDeckList()
        self.assertTrue(isinstance(deck_list, list))

    @defer.inlineCallbacks
    def test_get_deck_descriptor(self):
        deck_descriptor = yield self.oonibclient.getDeck(deck_id)
        for key in ['name', 'description', 'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(deck_descriptor, key))

    @defer.inlineCallbacks
    def test_download_deck(self):
        yield self.oonibclient.downloadDeck(deck_id)

    def test_lookup_invalid_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        return self.failUnlessFailure(
            self.oonibclient.lookupTestHelpers(['sdadsadsa', 'dns']),
            e.CouldNotFindTestHelper)

    @defer.inlineCallbacks
    def test_lookup_no_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = []
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertTrue('default' in helpers.keys())

    @defer.inlineCallbacks
    def test_lookup_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = [u'http-return-json-headers', u'dns']
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertEqual(set(helpers.keys()),
                         set(required_helpers + [u'default']))
        self.assertTrue(
            helpers['http-return-json-headers']['address'].startswith('http'))
        self.assertTrue(int(helpers['dns']['address'].split('.')[0]))

    @defer.inlineCallbacks
    def test_input_descriptor_not_found(self):
        yield self.assertFailure(
            self.oonibclient.queryBackend('GET', '/input/' + 'a' * 64),
            e.OONIBInputDescriptorNotFound)

    @defer.inlineCallbacks
    def test_http_errors(self):
        yield self.assertFailure(
            self.oonibclient.queryBackend('PUT', '/policy/input'), error.Error)

    @defer.inlineCallbacks
    def test_create_report(self):
        res = yield self.oonibclient.queryBackend(
            'POST', '/report', {
                'software_name': 'spam',
                'software_version': '2.0',
                'probe_asn': 'AS0',
                'probe_cc': 'ZZ',
                'test_name': 'foobar',
                'test_version': '1.0',
                'input_hashes': []
            })
        assert isinstance(res['report_id'], unicode)

    @defer.inlineCallbacks
    def test_report_lifecycle(self):
        res = yield self.oonibclient.queryBackend(
            'POST', '/report', {
                'software_name': 'spam',
                'software_version': '2.0',
                'probe_asn': 'AS0',
                'probe_cc': 'ZZ',
                'test_name': 'foobar',
                'test_version': '1.0',
                'input_hashes': []
            })
        report_id = str(res['report_id'])

        res = yield self.oonibclient.queryBackend(
            'POST', '/report/' + report_id,
            {'content': '---\nspam: ham\n...\n'})

        res = yield self.oonibclient.queryBackend(
            'POST', '/report/' + report_id,
            {'content': '---\nspam: ham\n...\n'})

        res = yield self.oonibclient.queryBackend(
            'POST', '/report/' + report_id + '/close')
Esempio n. 16
0
class TestOONIBClient(unittest.TestCase):
    def setUp(self):
        host = '127.0.0.1'
        port = 8889
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host, port))
            s.shutdown(2)
            try: shutil.rmtree(data_dir)
            except: pass
            os.mkdir(data_dir)
            os.mkdir(os.path.join(data_dir, 'inputs'))
            os.mkdir(os.path.join(data_dir, 'decks'))
        except Exception as ex:
            self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
    
    @defer.inlineCallbacks
    def test_query(self):
        res = yield self.oonibclient.queryBackend('GET', '/policy/input')
        self.assertTrue(isinstance(res, list))
    
    @defer.inlineCallbacks
    def test_get_input_list(self):
        input_list = yield self.oonibclient.getInputList()
        self.assertTrue(isinstance(input_list, list))

    @defer.inlineCallbacks
    def test_get_input_descriptor(self):
        input_descriptor = yield self.oonibclient.getInput(input_id)
        for key in ['name', 'description', 
                    'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(input_descriptor, key))

    @defer.inlineCallbacks
    def test_download_input(self):
        yield self.oonibclient.downloadInput(input_id)

    @defer.inlineCallbacks
    def test_get_deck_list(self):
        deck_list = yield self.oonibclient.getDeckList()
        self.assertTrue(isinstance(deck_list, list))

    @defer.inlineCallbacks
    def test_get_deck_descriptor(self):
        deck_descriptor = yield self.oonibclient.getDeck(deck_id)
        for key in ['name', 'description', 
                    'version', 'author', 'date', 'id']:
            self.assertTrue(hasattr(deck_descriptor, key))

    @defer.inlineCallbacks
    def test_download_deck(self):
        yield self.oonibclient.downloadDeck(deck_id)

    def test_lookup_invalid_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        return self.failUnlessFailure(
                self.oonibclient.lookupTestHelpers([
                    'sdadsadsa', 'dns'
                ]), e.CouldNotFindTestHelper)

    @defer.inlineCallbacks
    def test_lookup_no_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = []
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertTrue('default' in helpers.keys())

    @defer.inlineCallbacks
    def test_lookup_test_helpers(self):
        self.oonibclient.address = 'http://127.0.0.1:8888'
        required_helpers = [u'http-return-json-headers', u'dns']
        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
        self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))
        self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http'))
        self.assertTrue(int(helpers['dns']['address'].split('.')[0]))

    @defer.inlineCallbacks
    def test_invalid_requests(self):

        @defer.inlineCallbacks
        def all_requests(path):
            for mthd in ['GET', 'POST', 'PUT', 'OPTION']:
                try:
                    yield self.oonibclient.queryBackend(mthd, path)
                except:
                    pass

        for path in ['/policy/input', '/policy/nettest', 
                '/input', '/input/'+'a'*64, '/fooo']:
            yield all_requests(path)

        for path in ['/bouncer']:
            self.oonibclient.address = 'http://127.0.0.1:8888'
            yield all_requests(path)