def setupCollector(tor_process_protocol): def setup_complete(port): print("Exposed collector Tor hidden service on httpo://%s" % port.onion_uri) tempfile.tempdir = os.path.join(_repo_dir, 'tmp') if not os.path.isdir(tempfile.gettempdir()): os.makedirs(tempfile.gettempdir()) _temp_dir = tempfile.mkdtemp() if config.main.tor_datadir is None: log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") log.msg("Creating tmp directory in current directory for datadir.") log.debug("Using %s" % _temp_dir) datadir = _temp_dir else: datadir = config.main.tor_datadir torconfig = TorConfig(tor_process_protocol.tor_protocol) public_port = 80 # XXX there is currently a bug in txtorcon that prevents data_dir from # being passed properly. Details on the bug can be found here: # https://github.com/meejah/txtorcon/pull/22 hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, data_dir=datadir) hidden_service = hs_endpoint.listen(reportingBackend) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed)
def startTor(): def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) tempfile.tempdir = os.path.join(_repo_dir, 'tmp') if not os.path.isdir(tempfile.gettempdir()): os.makedirs(tempfile.gettempdir()) _temp_dir = tempfile.mkdtemp() torconfig = TorConfig() torconfig.SocksPort = config.main.socks_port if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60 if config.main.tor_datadir is None: log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") log.msg("Creating tmp directory in current directory for datadir.") log.debug("Using %s" % _temp_dir) datadir = _temp_dir else: datadir = config.main.tor_datadir torconfig.DataDirectory = datadir torconfig.save() if config.main.tor_binary is not None: d = launch_tor(torconfig, reactor, tor_binary=config.main.tor_binary, progress_updates=updates) else: d = launch_tor(torconfig, reactor, progress_updates=updates) d.addCallback(setupCollector, datadir) if ooniBouncer: d.addCallback(setupBouncer, datadir) d.addErrback(txSetupFailed)
def postApplication(self): """After the application is created, start the application and run the reactor. After the reactor stops, clean up PID files and such. """ self.startApplication(self.application) # This is our addition. The rest is taken from # twisted/scripts/_twistd_unix.py 12.2.0 startTor() self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if os.path.exists(tempfile.gettempdir()): log.msg("Removing temporary directory: %s" % tempfile.gettempdir()) rmtree(tempfile.gettempdir(), onerror=log.err)
def log_function(handler): values = _LaxDict({ 'request_time': 1000.0 * handler.request.request_time(), 'protocol': handler.request.protocol, 'status': str(handler.get_status()), 'request_method': handler.request.method, 'request_uri': handler.request.uri, 'remote_ip': handler.request.remote_ip }) log_format = config.main.log_format if not log_format: log_format = ("[{protocol}] {status} {request_method} {request_uri} " "127.0.0.1 {request_time}ms") log.msg(log_format.format(**values))
def post(self): try: request = json.loads(self.request.body) self.validate_request(request) include_http_responses = request.get("include_http_responses", False) self.control_measurement( str(request['http_request']), request['tcp_connect'], include_http_responses ) except HTTPError: raise except Exception as exc: log.msg("Got invalid request") log.exception(exc) raise HTTPError(400, 'invalid request')
def post(self): try: request = json.loads(self.request.body) # Here we fix inconsistencies in the tcp_connect field. Due to: # https://github.com/TheTorProject/ooni-probe/issues/727 ooniprobe # was sending hostnames as part of the tcp_connect key as well as # IP addresses. # If we find something that isn't an IP address we return it with # the key value None to support backward compatibility with older # bugged clients. tcp_connect = [] invalid_sockets = [] for socket in request['tcp_connect']: if SOCKET_REGEXP.match(socket): tcp_connect.append(socket) else: invalid_sockets.append(socket) request['tcp_connect'] = tcp_connect self.validate_request(request) include_http_responses = request.get( "include_http_responses", False ) # We convert headers to str so twisted is happy (unicode triggers # errors) http_request_headers = {} for k, v in request.get('http_request_headers', {}).iteritems(): http_request_headers[str(k)] = map(str, v) self.control_measurement( http_url=str(request['http_request']), include_http_responses=include_http_responses, http_request_headers=http_request_headers, socket_list=request['tcp_connect'], invalid_sockets=invalid_sockets ) except HTTPError: raise except Exception as exc: log.msg("Got invalid request") log.exception(exc) raise HTTPError(400, 'invalid request')
def postApplication(self): """After the application is created, start the application and run the reactor. After the reactor stops, clean up PID files and such. """ self.startApplication(self.application) # This is our addition. The rest is taken from # twisted/scripts/_twistd_unix.py 12.2.0 if config.main.tor_hidden_service: startTor() else: if ooniBouncer: reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1") reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1") self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if os.path.exists(tempfile.gettempdir()): log.msg("Removing temporary directory: %s" % tempfile.gettempdir()) rmtree(tempfile.gettempdir(), onerror=log.err)
def postApplication(self): """After the application is created, start the application and run the reactor. After the reactor stops, clean up PID files and such. """ self.startApplication(self.application) # This is our addition. The rest is taken from # twisted/scripts/_twistd_unix.py 12.2.0 if config.main.tor_hidden_service: torconfig = TorConfig() d = self.startTor(torconfig) d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend) if ooniBouncer: d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer) else: if ooniBouncer: reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1") reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1") self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if self.temporary_data_dir: log.msg("Removing temporary directory: %s" % self.temporary_data_dir) rmtree(self.temporary_data_dir, onerror=log.err)
def nextStep(self): log.debug("Moving on to next step in the state walk") self.current_data_received = 0 # Python why? if self.current_step >= (len(self.steps) - 1): log.msg("Reached the end of the state machine") log.msg("Censorship fingerpint bisected!") step_idx, mutation_idx = self.factory.mutation log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) # self.transport.loseConnection() if self.report: self.report["mutation_idx"] = mutation_idx self.report["step_idx"] = step_idx return else: self.current_step += 1 if self._current_step_role() == self.role: # We need to send more data because we are again responsible for # doing so. self.sendPayload()
def nextStep(self): log.debug("Moving on to next step in the state walk") self.current_data_received = 0 # Python why? if self.current_step >= (len(self.steps) - 1): log.msg("Reached the end of the state machine") log.msg("Censorship fingerpint bisected!") step_idx, mutation_idx = self.factory.mutation log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) #self.transport.loseConnection() if self.report: self.report['mutation_idx'] = mutation_idx self.report['step_idx'] = step_idx return else: self.current_step += 1 if self._current_step_role() == self.role: # We need to send more data because we are again responsible for # doing so. self.sendPayload()
def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary))