def _setUp(self): log.debug("Setting up HTTPTest") try: import OpenSSL except: log.err("Warning! pyOpenSSL is not installed. https websites will" "not work") self.control_agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.advanced.tor_socksport) sockshost, socksport = (None, None) if self.localOptions['socksproxy']: self.report['socksproxy'] = self.localOptions['socksproxy'] sockshost, socksport = self.localOptions['socksproxy'].split(':') socksport = int(socksport) self.agent = Agent(reactor, sockshost=sockshost, socksport=socksport) if self.followRedirects: try: from twisted.web.client import RedirectAgent self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) except: log.err("Warning! You are running an old version of twisted"\ "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") self.report['errors'].append("Could not import RedirectAgent") self.processInputs() log.debug("Finished test setup")
def __init__(self, backend_url): from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=int(config.advanced.tor_socksport)) except Exception, e: log.exception(e)
def _setUp(self): super(HTTPTest, self)._setUp() try: import OpenSSL except: log.err("Warning! pyOpenSSL is not installed. https websites will " "not work") self.control_agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port) self.report['socksproxy'] = None sockshost, socksport = (None, None) if self.localOptions['socksproxy']: try: sockshost, socksport = self.localOptions['socksproxy'].split(':') self.report['socksproxy'] = self.localOptions['socksproxy'] except ValueError: raise InvalidSocksProxyOption socksport = int(socksport) self.agent = Agent(reactor, sockshost=sockshost, socksport=socksport) self.report['agent'] = 'agent' if self.followRedirects: try: from twisted.web.client import RedirectAgent self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted"\ "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") self.processInputs() log.debug("Finished test setup")
def createReport(self): """ Creates a report on the oonib collector. """ # XXX we should probably be setting this inside of the constructor, # however config.tor.socks_port is not set until Tor is started and the # reporter is instantiated before Tor is started. We probably want to # do this with some deferred kung foo or instantiate the reporter after # tor is started. from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=int(config.tor.socks_port)) except Exception, e: log.exception(e)
def __init__(self, address): self.address = address self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port)
class OONIBClient(object): retries = 3 def __init__(self, address): self.address = address self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port) def _request(self, method, urn, genReceiver, bodyProducer=None): attempts = 0 finished = defer.Deferred() def perform_request(): uri = self.address + urn headers = {} d = self.agent.request(method, uri, bodyProducer=bodyProducer) @d.addCallback def callback(response): content_length = int(response.headers.getRawHeaders('content-length')[0]) response.deliverBody(genReceiver(finished, content_length)) def errback(err, attempts): # We we will recursively keep trying to perform a request until # we have reached the retry count. if attempts < self.retries: log.err("Lookup failed. Retrying.") attempts += 1 perform_request() else: log.err("Failed. Giving up.") finished.errback(err) d.addErrback(errback, attempts) perform_request() return finished def queryBackend(self, method, urn, query=None): bodyProducer = None if query: bodyProducer = StringProducer(json.dumps(query)) def genReceiver(finished, content_length): def process_response(s): try: response = json.loads(s) except ValueError: raise e.get_error(None) if 'error' in response: print "Got this backend error message %s" % response log.err("Got this backend error message %s" % response) raise e.get_error(response['error']) return response return BodyReceiver(finished, content_length, process_response) return self._request(method, urn, genReceiver, bodyProducer) def download(self, urn, download_path): def genReceiver(finished, content_length): return Downloader(download_path, finished, content_length) return self._request('GET', urn, genReceiver) def getNettestPolicy(self): pass def getInput(self, input_hash): input_file = InputFile(input_hash) if input_file.descriptorCached: return defer.succeed(input_file) else: d = self.queryBackend('GET', '/input/' + input_hash) @d.addCallback def cb(descriptor): input_file.load(descriptor) input_file.save() return input_file @d.addErrback def err(err): log.err("Failed to get descriptor for input %s" % input_hash) log.exception(err) return d def getInputList(self): return self.queryBackend('GET', '/input') def downloadInput(self, input_hash): input_file = InputFile(input_hash) if input_file.fileCached: return defer.succeed(input_file) else: d = self.download('/input/'+input_hash+'/file', input_file.cached_file) @d.addCallback def cb(res): input_file.verify() return input_file @d.addErrback def err(err): log.err("Failed to download the input file %s" % input_hash) log.exception(err) return d def getInputPolicy(self): return self.queryBackend('GET', '/policy/input') def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest') def getDeckList(self): return self.queryBackend('GET', '/deck') def getDeck(self, deck_hash): deck = Deck(deck_hash) if deck.descriptorCached: return defer.succeed(deck) else: d = self.queryBackend('GET', '/deck/' + deck_hash) @d.addCallback def cb(descriptor): deck.load(descriptor) deck.save() return deck @d.addErrback def err(err): log.err("Failed to get descriptor for deck %s" % deck_hash) print err log.exception(err) return d def downloadDeck(self, deck_hash): deck = Deck(deck_hash) if deck.fileCached: return defer.succeed(deck) else: d = self.download('/deck/'+deck_hash+'/file', deck.cached_file) @d.addCallback def cb(res): deck.verify() return deck @d.addErrback def err(err): log.err("Failed to download the deck %s" % deck_hash) print err log.exception(err) return d @defer.inlineCallbacks def lookupTestCollector(self, test_name): try: test_collector = yield self.queryBackend('POST', '/bouncer', query={'test-collector': test_name}) except Exception: raise e.CouldNotFindTestCollector defer.returnValue(test_collector) @defer.inlineCallbacks def lookupTestHelpers(self, test_helper_names): try: test_helper = yield self.queryBackend('POST', '/bouncer', query={'test-helpers': test_helper_names}) except Exception, e: log.exception(e) raise e.CouldNotFindTestHelper if not test_helper: raise e.CouldNotFindTestHelper defer.returnValue(test_helper)