def setupCollector(tor_process_protocol):
def setup_complete(port):
print("Exposed collector Tor hidden service on httpo://%s"
% port.onion_uri)
tempfile.tempdir = os.path.join(_repo_dir, 'tmp')
if not os.path.isdir(tempfile.gettempdir()):
os.makedirs(tempfile.gettempdir())
_temp_dir = tempfile.mkdtemp()
if config.main.tor_datadir is None:
log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
log.msg("Creating tmp directory in current directory for datadir.")
log.debug("Using %s" % _temp_dir)
datadir = _temp_dir
else:
datadir = config.main.tor_datadir
torconfig = TorConfig(tor_process_protocol.tor_protocol)
public_port = 80
# XXX there is currently a bug in txtorcon that prevents data_dir from
# being passed properly. Details on the bug can be found here:
# https://github.com/meejah/txtorcon/pull/22
hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port,
data_dir=datadir)
hidden_service = hs_endpoint.listen(reportingBackend)
hidden_service.addCallback(setup_complete)
hidden_service.addErrback(txSetupFailed)
def startTor():
def updates(prog, tag, summary):
print("%d%%: %s" % (prog, summary))
tempfile.tempdir = os.path.join(_repo_dir, 'tmp')
if not os.path.isdir(tempfile.gettempdir()):
os.makedirs(tempfile.gettempdir())
_temp_dir = tempfile.mkdtemp()
torconfig = TorConfig()
torconfig.SocksPort = config.main.socks_port
if config.main.tor2webmode:
torconfig.Tor2webMode = 1
torconfig.CircuitBuildTimeout = 60
if config.main.tor_datadir is None:
log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
log.msg("Creating tmp directory in current directory for datadir.")
log.debug("Using %s" % _temp_dir)
datadir = _temp_dir
else:
datadir = config.main.tor_datadir
torconfig.DataDirectory = datadir
torconfig.save()
if config.main.tor_binary is not None:
d = launch_tor(torconfig, reactor,
tor_binary=config.main.tor_binary,
progress_updates=updates)
else:
d = launch_tor(torconfig, reactor, progress_updates=updates)
d.addCallback(setupCollector, datadir)
if ooniBouncer:
d.addCallback(setupBouncer, datadir)
d.addErrback(txSetupFailed)
def postApplication(self):
"""After the application is created, start the application and run
the reactor. After the reactor stops, clean up PID files and such.
"""
self.startApplication(self.application)
# This is our addition. The rest is taken from
# twisted/scripts/_twistd_unix.py 12.2.0
startTor()
self.startReactor(None, self.oldstdout, self.oldstderr)
self.removePID(self.config['pidfile'])
if os.path.exists(tempfile.gettempdir()):
log.msg("Removing temporary directory: %s"
% tempfile.gettempdir())
rmtree(tempfile.gettempdir(), onerror=log.err)
def log_function(handler):
values = _LaxDict({
'request_time': 1000.0 * handler.request.request_time(),
'protocol': handler.request.protocol,
'status': str(handler.get_status()),
'request_method': handler.request.method,
'request_uri': handler.request.uri,
'remote_ip': handler.request.remote_ip
})
log_format = config.main.log_format
if not log_format:
log_format = ("[{protocol}] {status} {request_method} {request_uri} "
"127.0.0.1 {request_time}ms")
log.msg(log_format.format(**values))
def log_function(handler):
values = _LaxDict({
'request_time': 1000.0 * handler.request.request_time(),
'protocol': handler.request.protocol,
'status': str(handler.get_status()),
'request_method': handler.request.method,
'request_uri': handler.request.uri,
'remote_ip': handler.request.remote_ip
})
log_format = config.main.log_format
if not log_format:
log_format = ("[{protocol}] {status} {request_method} {request_uri} "
"127.0.0.1 {request_time}ms")
log.msg(log_format.format(**values))
def post(self):
try:
request = json.loads(self.request.body)
self.validate_request(request)
include_http_responses = request.get("include_http_responses",
False)
self.control_measurement(
str(request['http_request']),
request['tcp_connect'],
include_http_responses
)
except HTTPError:
raise
except Exception as exc:
log.msg("Got invalid request")
log.exception(exc)
raise HTTPError(400, 'invalid request')
def post(self):
try:
request = json.loads(self.request.body)
# Here we fix inconsistencies in the tcp_connect field. Due to:
# https://github.com/TheTorProject/ooni-probe/issues/727 ooniprobe
# was sending hostnames as part of the tcp_connect key as well as
# IP addresses.
# If we find something that isn't an IP address we return it with
# the key value None to support backward compatibility with older
# bugged clients.
tcp_connect = []
invalid_sockets = []
for socket in request['tcp_connect']:
if SOCKET_REGEXP.match(socket):
tcp_connect.append(socket)
else:
invalid_sockets.append(socket)
request['tcp_connect'] = tcp_connect
self.validate_request(request)
include_http_responses = request.get(
"include_http_responses",
False
)
# We convert headers to str so twisted is happy (unicode triggers
# errors)
http_request_headers = {}
for k, v in request.get('http_request_headers', {}).iteritems():
http_request_headers[str(k)] = map(str, v)
self.control_measurement(
http_url=str(request['http_request']),
include_http_responses=include_http_responses,
http_request_headers=http_request_headers,
socket_list=request['tcp_connect'],
invalid_sockets=invalid_sockets
)
except HTTPError:
raise
except Exception as exc:
log.msg("Got invalid request")
log.exception(exc)
raise HTTPError(400, 'invalid request')
def postApplication(self):
"""After the application is created, start the application and run
the reactor. After the reactor stops, clean up PID files and such.
"""
self.startApplication(self.application)
# This is our addition. The rest is taken from
# twisted/scripts/_twistd_unix.py 12.2.0
if config.main.tor_hidden_service:
startTor()
else:
if ooniBouncer:
reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1")
reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1")
self.startReactor(None, self.oldstdout, self.oldstderr)
self.removePID(self.config['pidfile'])
if os.path.exists(tempfile.gettempdir()):
log.msg("Removing temporary directory: %s"
% tempfile.gettempdir())
rmtree(tempfile.gettempdir(), onerror=log.err)
def postApplication(self):
"""After the application is created, start the application and run
the reactor. After the reactor stops, clean up PID files and such.
"""
self.startApplication(self.application)
# This is our addition. The rest is taken from
# twisted/scripts/_twistd_unix.py 12.2.0
if config.main.tor_hidden_service:
torconfig = TorConfig()
d = self.startTor(torconfig)
d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend)
if ooniBouncer:
d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer)
else:
if ooniBouncer:
reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1")
reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1")
self.startReactor(None, self.oldstdout, self.oldstderr)
self.removePID(self.config['pidfile'])
if self.temporary_data_dir:
log.msg("Removing temporary directory: %s"
% self.temporary_data_dir)
rmtree(self.temporary_data_dir, onerror=log.err)
def postApplication(self):
"""After the application is created, start the application and run
the reactor. After the reactor stops, clean up PID files and such.
"""
self.startApplication(self.application)
# This is our addition. The rest is taken from
# twisted/scripts/_twistd_unix.py 12.2.0
if config.main.tor_hidden_service:
torconfig = TorConfig()
d = self.startTor(torconfig)
d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend)
if ooniBouncer:
d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer)
else:
if ooniBouncer:
reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1")
reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1")
self.startReactor(None, self.oldstdout, self.oldstderr)
self.removePID(self.config['pidfile'])
if self.temporary_data_dir:
log.msg("Removing temporary directory: %s"
% self.temporary_data_dir)
rmtree(self.temporary_data_dir, onerror=log.err)
def nextStep(self):
log.debug("Moving on to next step in the state walk")
self.current_data_received = 0
# Python why?
if self.current_step >= (len(self.steps) - 1):
log.msg("Reached the end of the state machine")
log.msg("Censorship fingerpint bisected!")
step_idx, mutation_idx = self.factory.mutation
log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx))
# self.transport.loseConnection()
if self.report:
self.report["mutation_idx"] = mutation_idx
self.report["step_idx"] = step_idx
return
else:
self.current_step += 1
if self._current_step_role() == self.role:
# We need to send more data because we are again responsible for
# doing so.
self.sendPayload()
def nextStep(self):
log.debug("Moving on to next step in the state walk")
self.current_data_received = 0
# Python why?
if self.current_step >= (len(self.steps) - 1):
log.msg("Reached the end of the state machine")
log.msg("Censorship fingerpint bisected!")
step_idx, mutation_idx = self.factory.mutation
log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx))
#self.transport.loseConnection()
if self.report:
self.report['mutation_idx'] = mutation_idx
self.report['step_idx'] = step_idx
return
else:
self.current_step += 1
if self._current_step_role() == self.role:
# We need to send more data because we are again responsible for
# doing so.
self.sendPayload()
def updates(prog, tag, summary):
log.msg("%d%%: %s" % (prog, summary))
def updates(prog, tag, summary):
log.msg("%d%%: %s" % (prog, summary))
