def run_tor(cmdline):
"""Run the tor command line cmdline, which must start with the path or
name of a tor binary.
Returns the combined stdout and stderr of the process.
"""
if not debug_flag:
cmdline.append("--quiet")
try:
stdouterr = subprocess.check_output(cmdline,
stderr=subprocess.STDOUT,
universal_newlines=True,
bufsize=-1)
debug(stdouterr)
except OSError as e:
# only catch file not found error
if e.errno == errno.ENOENT:
_warnMissingTor(cmdline[0], cmdline)
sys.exit(1)
else:
raise
except subprocess.CalledProcessError as e:
# only catch file not found error
if e.returncode == 127:
_warnMissingTor(cmdline[0], cmdline)
sys.exit(1)
else:
raise
return stdouterr
def run(self):
start = now = time.time()
end = time.time() + self.timeout
DUMP_TEST_STATUS_INTERVAL=0.5
dump_at = start+DUMP_TEST_STATUS_INTERVAL
while now < end and not self.tests.all_done():
# run only one iteration at a time, with a nice short timeout, so we
# can actually detect completion and timeouts.
asyncore.loop(5.0, False, self.socket_map, 1)
now = time.time()
if now > dump_at:
debug("Test status: %s"%self.tests.status())
dump_at += DUMP_TEST_STATUS_INTERVAL
if not debug_flag:
sys.stdout.write('\n')
sys.stdout.flush()
debug("Done with run(); all_done == %s and failure_count == %s"
%(self.tests.all_done(), self.tests.failure_count()))
note("Status:\n%s"%self.tests.teststatus)
self.listener.close()
return self.tests.all_done() and self.tests.failure_count() == 0
def _genAuthorityKey(self):
"""Generate an authority identity and signing key for this authority,
if they do not already exist."""
datadir = self._env['dir']
tor_gencert = self._env['tor_gencert']
lifetime = self._env['auth_cert_lifetime']
idfile = os.path.join(datadir, 'keys', "authority_identity_key")
skfile = os.path.join(datadir, 'keys', "authority_signing_key")
certfile = os.path.join(datadir, 'keys', "authority_certificate")
addr = self.expand("${ip}:${dirport}")
passphrase = self._env['auth_passphrase']
if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
return
cmdline = [
tor_gencert,
'--create-identity-key',
'--passphrase-fd', '0',
'-i', idfile,
'-s', skfile,
'-c', certfile,
'-m', str(lifetime),
'-a', addr,
]
# nicknames are testNNNaa[OLD], but we want them to look tidy
print("Creating identity key for {:12} with {}"
.format(self._env['nick'], cmdline[0]))
debug("Identity key path '{}', command '{}'"
.format(idfile, " ".join(cmdline)))
run_tor_gencert(cmdline, passphrase)
def _genAuthorityKey(self):
"""Generate an authority identity and signing key for this authority,
if they do not already exist."""
datadir = self._env['dir']
tor_gencert = self._env['tor_gencert']
lifetime = self._env['auth_cert_lifetime']
idfile = os.path.join(datadir, 'keys', "authority_identity_key")
skfile = os.path.join(datadir, 'keys', "authority_signing_key")
certfile = os.path.join(datadir, 'keys', "authority_certificate")
addr = self.expand("${ip}:${dirport}")
passphrase = self._env['auth_passphrase']
if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
return
cmdline = [
tor_gencert,
'--create-identity-key',
'--passphrase-fd',
'0',
'-i',
idfile,
'-s',
skfile,
'-c',
certfile,
'-m',
str(lifetime),
'-a',
addr,
]
# nicknames are testNNNaa[OLD], but we want them to look tidy
print("Creating identity key for {:12} with {}".format(
self._env['nick'], cmdline[0]))
debug("Identity key path '{}', command '{}'".format(
idfile, " ".join(cmdline)))
run_tor_gencert(cmdline, passphrase)
def run_tor(cmdline):
"""Run the tor command line cmdline, which must start with the path or
name of a tor binary.
Returns the combined stdout and stderr of the process.
"""
if not debug_flag:
cmdline.append("--quiet")
try:
stdouterr = subprocess.check_output(cmdline,
stderr=subprocess.STDOUT,
universal_newlines=True,
bufsize=-1)
debug(stdouterr)
except OSError as e:
# only catch file not found error
if e.errno == errno.ENOENT:
_warnMissingTor(cmdline[0], cmdline)
sys.exit(1)
else:
raise
except subprocess.CalledProcessError as e:
# only catch file not found error
if e.returncode == 127:
_warnMissingTor(cmdline[0], cmdline)
sys.exit(1)
else:
raise
return stdouterr
def __init__(self,
endpoint,
data=b"",
timeout=3,
repetitions=1,
dot_repetitions=0,
chat_type="Echo"):
if chat_type == "Echo":
self.client_class = EchoClient
self.responder_class = EchoServer
else:
self.client_class = Source
self.responder_class = Sink
self.socket_map = {}
self.listener = Listener(self, endpoint)
self.pending_close = []
self.timeout = timeout
self.tests = TestSuite()
self.data_source = DataSource(data, repetitions)
# sanity checks
self.dot_repetitions = dot_repetitions
debug("listener fd=%d" % self.listener.fileno())
def handle_accept(self):
# deprecated in python 3.2
pair = self.accept()
if pair is not None:
newsock, endpoint = pair
debug("new client from %s:%s (fd=%d)" %
(endpoint[0], endpoint[1], newsock.fileno()))
self.tt.add_responder(newsock)
def add(self, name):
note("Registering %s"%name)
if name not in self.tests:
debug("Registering %s"%name)
self.not_done += 1
self.tests[name] = 'not done'
else:
warn("... already registered!")
def failure(self, name):
note("Failure for %s"%name)
if self.tests[name] == 'not done':
debug("Failed %s"%name)
self.tests[name] = 'failure'
self.not_done -= 1
self.failures += 1
else:
warn("... status was %s"%self.tests.get(name))
def success(self, name):
note("Success for %s"%name)
if self.tests[name] == 'not done':
debug("Succeeded %s"%name)
self.tests[name] = 'success'
self.not_done -= 1
self.successes += 1
else:
warn("... status was %s"%self.tests.get(name))
def connect(self, endpoint):
self.dest = endpoint
self.state = self.CONNECTING
dest = self.proxy or self.dest
try:
debug("socket %d connecting to %r..." % (self.fd(), dest))
self.s.connect(dest)
except socket.error as e:
if e[0] != errno.EINPROGRESS:
raise
def connect(self, endpoint):
self.dest = endpoint
self.state = self.CONNECTING
dest = self.proxy or self.dest
try:
debug("socket %d connecting to %r..."%(self.fd(),dest))
self.s.connect(dest)
except socket.error as e:
if e.errno != errno.EINPROGRESS:
raise
def run_tor_gencert(cmdline, passphrase):
"""Run the tor-gencert command line cmdline, which must start with the
path or name of a tor-gencert binary.
Then send passphrase to the stdin of the process.
Returns the combined stdout and stderr of the process.
"""
p = launch_process(cmdline, tor_name="tor-gencert", stdin=subprocess.PIPE)
(stdouterr, empty_stderr) = p.communicate(passphrase + "\n")
debug(stdouterr)
assert p.returncode == 0 # XXXX BAD!
assert empty_stderr is None
return stdouterr
def start(self):
"""Try to start this node; return True if we succeeded or it was
already running, False if we failed."""
if self.isRunning():
print("{:12} is already running".format(self._env['nick']))
return True
tor_path = self._env['tor']
torrc = self._getTorrcFname()
cmdline = [
tor_path,
"-f", torrc,
]
p = launch_process(cmdline)
if self.waitOnLaunch():
# this requires that RunAsDaemon is set
(stdouterr, empty_stderr) = p.communicate()
debug(stdouterr)
assert empty_stderr is None
else:
# this does not require RunAsDaemon to be set, but is slower.
#
# poll() only catches failures before the call itself
# so let's sleep a little first
# this does, of course, slow down process launch
# which can require an adjustment to the voting interval
#
# avoid writing a newline or space when polling
# so output comes out neatly
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(self._env['poll_launch_time'])
p.poll()
if p.returncode is not None and p.returncode != 0:
if self._env['poll_launch_time'] is None:
print(("Couldn't launch {:12} command '{}': " +
"exit {}, output '{}'")
.format(self._env['nick'],
" ".join(cmdline),
p.returncode,
stdouterr))
else:
print(("Couldn't poll {:12} command '{}' " +
"after waiting {} seconds for launch: " +
"exit {}").format(self._env['nick'],
" ".join(cmdline),
self._env['poll_launch_time'],
p.returncode))
return False
return True
def run_tor_gencert(cmdline, passphrase):
"""Run the tor-gencert command line cmdline, which must start with the
path or name of a tor-gencert binary.
Then send passphrase to the stdin of the process.
Returns the combined stdout and stderr of the process.
"""
p = launch_process(cmdline,
tor_name="tor-gencert",
stdin=subprocess.PIPE)
(stdouterr, empty_stderr) = p.communicate(passphrase + "\n")
debug(stdouterr)
assert p.returncode == 0 # XXXX BAD!
assert empty_stderr is None
return stdouterr
def __init__(self, tt, server, proxy=None):
asynchat.async_chat.__init__(self, map=tt.socket_map)
self.data_source = tt.data_source.copy()
self.inbuf = b''
self.proxy = proxy
self.server = server
self.tt = tt
self.testname = uniq("send-data")
self.set_terminator(None)
dest = (self.proxy or self.server)
self.create_socket(addr_to_family(dest[0]), socket.SOCK_STREAM)
debug("socket %d connecting to %r..."%(self.fileno(),dest))
self.state = self.CONNECTING
self.connect(dest)
def collect_incoming_data(self, data):
self.inbuf += data
if self.state == self.CONNECTING_THROUGH_PROXY:
if len(self.inbuf) >= 8:
if self.inbuf[:2] == b'\x00\x5a':
self.note("proxy handshake successful")
self.state = self.CONNECTED
debug("successfully connected (fd=%d)" % self.fileno())
self.inbuf = self.inbuf[8:]
self.push_output()
else:
debug("proxy handshake failed (0x%x)! (fd=%d)" %
(byte_to_int(self.inbuf[1]), self.fileno()))
self.state = self.NOT_CONNECTED
self.close()
def start(self):
"""Try to start this node; return True if we succeeded or it was
already running, False if we failed."""
if self.isRunning():
print("{:12} is already running".format(self._env['nick']))
return True
tor_path = self._env['tor']
torrc = self._getTorrcFname()
cmdline = [
tor_path,
"-f",
torrc,
]
p = launch_process(cmdline)
if self.waitOnLaunch():
# this requires that RunAsDaemon is set
(stdouterr, empty_stderr) = p.communicate()
debug(stdouterr)
assert empty_stderr is None
else:
# this does not require RunAsDaemon to be set, but is slower.
#
# poll() only catches failures before the call itself
# so let's sleep a little first
# this does, of course, slow down process launch
# which can require an adjustment to the voting interval
#
# avoid writing a newline or space when polling
# so output comes out neatly
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(self._env['poll_launch_time'])
p.poll()
if p.returncode is not None and p.returncode != 0:
if self._env['poll_launch_time'] is None:
print(("Couldn't launch {:12} command '{}': " +
"exit {}, output '{}'").format(self._env['nick'],
" ".join(cmdline),
p.returncode, stdouterr))
else:
print(("Couldn't poll {:12} command '{}' " +
"after waiting {} seconds for launch: " +
"exit {}").format(self._env['nick'], " ".join(cmdline),
self._env['poll_launch_time'],
p.returncode))
return False
return True
def _createTorrcFile(self, checkOnly=False):
"""Write the torrc file for this node, disabling any options
that are not supported by env's tor binary using comments.
If checkOnly, just make sure that the formatting is indeed
possible.
"""
global torrc_option_warn_count
fn_out = self._getTorrcFname()
torrc_template = self._getTorrcTemplate()
output = torrc_template.format(self._env)
if checkOnly:
# XXXX Is it time-consuming to format? If so, cache here.
return
# now filter the options we're about to write, commenting out
# the options that the current tor binary doesn't support
tor = self._env['tor']
tor_version = get_tor_version(tor)
torrc_opts = get_torrc_options(tor)
# check if each option is supported before writing it
# Unsupported option values may need special handling.
with open(fn_out, 'w') as f:
# we need to do case-insensitive option comparison
lower_opts = [opt.lower() for opt in torrc_opts]
# keep ends when splitting lines, so we can write them out
# using writelines() without messing around with "\n"s
for line in output.splitlines(True):
# check if the first word on the line is a supported option,
# preserving empty lines and comment lines
sline = line.strip()
if (len(sline) == 0 or
sline[0] == '#' or
sline.split()[0].lower() in lower_opts):
pass
else:
warn_msg = (("The tor binary at {} does not support " +
"the option in the torrc line:\n{}")
.format(tor, line.strip()))
if torrc_option_warn_count < TORRC_OPTION_WARN_LIMIT:
print(warn_msg)
torrc_option_warn_count += 1
else:
debug(warn_msg)
# always dump the full output to the torrc file
line = ("# {} version {} does not support: {}"
.format(tor, tor_version, line))
f.writelines([line])
def _createTorrcFile(self, checkOnly=False):
"""Write the torrc file for this node, disabling any options
that are not supported by env's tor binary using comments.
If checkOnly, just make sure that the formatting is indeed
possible.
"""
global torrc_option_warn_count
fn_out = self._getTorrcFname()
torrc_template = self._getTorrcTemplate()
output = torrc_template.format(self._env)
if checkOnly:
# XXXX Is it time-consuming to format? If so, cache here.
return
# now filter the options we're about to write, commenting out
# the options that the current tor binary doesn't support
tor = self._env['tor']
tor_version = get_tor_version(tor)
torrc_opts = get_torrc_options(tor)
# check if each option is supported before writing it
# Unsupported option values may need special handling.
with open(fn_out, 'w') as f:
# we need to do case-insensitive option comparison
lower_opts = [opt.lower() for opt in torrc_opts]
# keep ends when splitting lines, so we can write them out
# using writelines() without messing around with "\n"s
for line in output.splitlines(True):
# check if the first word on the line is a supported option,
# preserving empty lines and comment lines
sline = line.strip()
if (len(sline) == 0 or sline[0] == '#'
or sline.split()[0].lower() in lower_opts):
pass
else:
warn_msg = (("The tor binary at {} does not support " +
"the option in the torrc line:\n{}").format(
tor, line.strip()))
if torrc_option_warn_count < TORRC_OPTION_WARN_LIMIT:
print(warn_msg)
torrc_option_warn_count += 1
else:
debug(warn_msg)
# always dump the full output to the torrc file
line = ("# {} version {} does not support: {}".format(
tor, tor_version, line))
f.writelines([line])
def socks_cmd(addr_port):
"""
Return a SOCKS command for connecting to addr_port.
SOCKSv4: https://en.wikipedia.org/wiki/SOCKS#Protocol
SOCKSv5: RFC1928, RFC1929
"""
ver = 4 # Only SOCKSv4 for now.
cmd = 1 # Stream connection.
user = '******'
dnsname = ''
host, port = addr_port
try:
addr = socket.inet_aton(host)
except socket.error:
addr = '\x00\x00\x00\x01'
dnsname = '%s\x00' % host
debug("Socks 4a request to %s:%d" % (host, port))
return struct.pack('!BBH', ver, cmd, port) + addr + user + dnsname
def __init__(self,
endpoint,
data={},
timeout=3,
repetitions=1,
dot_repetitions=0):
self.listener = Listener(self, endpoint)
self.pending_close = []
self.timeout = timeout
self.tests = TestSuite()
self.data = data
self.repetitions = repetitions
# sanity checks
if len(self.data) == 0:
self.repetitions = 0
if self.repetitions == 0:
self.data = {}
self.dot_repetitions = dot_repetitions
debug("listener fd=%d" % self.listener.fd())
self.peers = {} # fd:Peer
def collect_incoming_data(self, data):
if self.state == self.CONNECTING_THROUGH_PROXY:
Source.collect_incoming_data(self, data)
if self.state == self.CONNECTING_THROUGH_PROXY:
return
data = self.inbuf
self.inbuf = b""
self.data_checker.consume(data)
self.enote("consumed some")
if self.data_checker.succeeded:
self.enote("successful verification")
debug("successful verification")
self.close()
self.tt.success(self.testname_check)
elif self.data_checker.failed:
debug("receive comparison failed")
self.tt.failure(self.testname_check)
self.close()
def __init__(self,
endpoint,
data={},
timeout=3,
repetitions=1,
dot_repetitions=0):
self.listener = Listener(self, endpoint)
self.pending_close = []
self.timeout = timeout
self.tests = TestSuite()
self.data = data
self.repetitions = repetitions
# sanity checks
if len(self.data) == 0:
self.repetitions = 0
if self.repetitions == 0:
self.data = {}
self.dot_repetitions = dot_repetitions
debug("listener fd=%d" % self.listener.fd())
self.peers = {} # fd:Peer
def _get_server_dns_resolv_conf(self, my):
if my['dns_conf'] == "":
# if the user asked for tor's default
return "#ServerDNSResolvConfFile using tor's compile-time default"
elif my['dns_conf'] is None:
# if there is no DNS conf file set
debug("CHUTNEY_DNS_CONF not specified, using '{}'.".format(
TorEnviron.DEFAULT_DNS_RESOLV_CONF))
dns_conf = TorEnviron.DEFAULT_DNS_RESOLV_CONF
else:
dns_conf = my['dns_conf']
dns_conf = os.path.abspath(dns_conf)
# work around Tor bug #21900, where exits fail when the DNS conf
# file does not exist, or is a broken symlink
# (os.path.exists returns False for broken symbolic links)
if not os.path.exists(dns_conf):
# Issue a warning so the user notices
print("CHUTNEY_DNS_CONF '{}' does not exist, using '{}'.".format(
dns_conf, TorEnviron.OFFLINE_DNS_RESOLV_CONF))
dns_conf = TorEnviron.OFFLINE_DNS_RESOLV_CONF
return "ServerDNSResolvConfFile %s" % (dns_conf)
def socks_cmd(addr_port):
"""
Return a SOCKS command for connecting to addr_port.
SOCKSv4: https://en.wikipedia.org/wiki/SOCKS#Protocol
SOCKSv5: RFC1928, RFC1929
"""
ver = 4 # Only SOCKSv4 for now.
cmd = 1 # Stream connection.
user = b'\x00'
dnsname = ''
host, port = addr_port
try:
addr = socket.inet_aton(host)
except socket.error:
addr = b'\x00\x00\x00\x01'
dnsname = '%s\x00' % host
debug("Socks 4a request to %s:%d" % (host, port))
if type(dnsname) != type(b""):
dnsname = dnsname.encode("ascii")
return struct.pack('!BBH', ver, cmd, port) + addr + user + dnsname
def _get_server_dns_resolv_conf(self, my):
if my['dns_conf'] == "":
# if the user asked for tor's default
return "#ServerDNSResolvConfFile using tor's compile-time default"
elif my['dns_conf'] is None:
# if there is no DNS conf file set
debug("CHUTNEY_DNS_CONF not specified, using '{}'."
.format(TorEnviron.DEFAULT_DNS_RESOLV_CONF))
dns_conf = TorEnviron.DEFAULT_DNS_RESOLV_CONF
else:
dns_conf = my['dns_conf']
dns_conf = os.path.abspath(dns_conf)
# work around Tor bug #21900, where exits fail when the DNS conf
# file does not exist, or is a broken symlink
# (os.path.exists returns False for broken symbolic links)
if not os.path.exists(dns_conf):
# Issue a warning so the user notices
print("CHUTNEY_DNS_CONF '{}' does not exist, using '{}'."
.format(dns_conf, TorEnviron.OFFLINE_DNS_RESOLV_CONF))
dns_conf = TorEnviron.OFFLINE_DNS_RESOLV_CONF
return "ServerDNSResolvConfFile %s" % (dns_conf)
def collect_incoming_data(self, inp):
# shortcut read when we don't ever expect any data
debug("successfully received (bytes=%d)" % len(inp))
self.data_checker.consume(inp)
if self.data_checker.succeeded:
debug("successful verification")
self.close()
self.tt.success(self.testname)
elif self.data_checker.failed:
debug("receive comparison failed")
self.tt.failure(self.testname)
self.close()
def on_writable(self):
"""Invoked when the socket becomes writable.
Return 0 when done writing
-1 on failure (like connection refused)
>0 if more data needs to be written
"""
if self.state == self.CONNECTING:
if self.proxy is None:
self.state = self.CONNECTED
debug("successfully connected (fd=%d)" % self.fd())
else:
self.state = self.CONNECTING_THROUGH_PROXY
self.outbuf = socks_cmd(self.dest)
# we write socks_cmd() to the proxy, then read the response
# if we get the correct response, we're CONNECTED
if self.state == self.CONNECTED:
# repeat self.data into self.outbuf if required
if (len(self.outbuf) < len(self.data) and self.repetitions > 0):
self.outbuf += self.data
self.repetitions -= 1
debug("adding more data to send (bytes=%d)" % len(self.data))
debug("now have data to send (bytes=%d)" % len(self.outbuf))
debug("send repetitions remaining (reps=%d)" %
self.repetitions)
try:
n = self.s.send(self.outbuf)
except socket.error as e:
if e[0] == errno.ECONNREFUSED:
debug("connection refused (fd=%d)" % self.fd())
return -1
raise
# sometimes, this debug statement prints 0
# it should print length of the data sent
# but the code works as long as this doesn't keep on happening
if n > 0:
debug("successfully sent (bytes=%d)" % n)
self._sent_no_bytes = 0
else:
debug("BUG: sent no bytes (out of %d; state is %s)" %
(len(self.outbuf), self.state))
self._sent_no_bytes += 1
# We can't retry too fast, otherwise clients burn all their HSDirs
if self._sent_no_bytes >= 2:
print("Sent no data %d times. Stalled." %
(self._sent_no_bytes))
return -1
time.sleep(5)
self.outbuf = self.outbuf[n:]
if self.state == self.CONNECTING_THROUGH_PROXY:
return 1 # Keep us around.
debug("bytes remaining on outbuf (bytes=%d)" % len(self.outbuf))
# calculate the actual length of data remaining, including reps
# When 0, we're being removed.
debug("bytes remaining overall (bytes=%d)" %
(self.repetitions * len(self.data) + len(self.outbuf)))
return self.repetitions * len(self.data) + len(self.outbuf)
def on_readable(self):
"""Invoked when the socket becomes readable.
Return -1 on failure
>0 if more data needs to be read or written
"""
if self.state == self.CONNECTING_THROUGH_PROXY:
inp = self.s.recv(8 - len(self.inbuf))
debug("-- connecting through proxy, got %d bytes" % len(inp))
if len(inp) == 0:
debug("EOF on fd %d" % self.fd())
return -1
self.inbuf += inp
if len(self.inbuf) == 8:
if ord(self.inbuf[0]) == 0 and ord(self.inbuf[1]) == 0x5a:
debug("proxy handshake successful (fd=%d)" % self.fd())
self.state = self.CONNECTED
self.inbuf = ''
debug("successfully connected (fd=%d)" % self.fd())
# if we have no reps or no data, skip sending actual data
if self.want_to_write():
return 1 # Keep us around for writing.
else:
# shortcut write when we don't ever expect any data
debug("no connection required - no data")
return 0
else:
debug("proxy handshake failed (0x%x)! (fd=%d)" %
(ord(self.inbuf[1]), self.fd()))
self.state = self.NOT_CONNECTED
return -1
assert (8 - len(self.inbuf) > 0)
return 8 - len(self.inbuf)
return self.want_to_write() # Keep us around for writing if needed
def cleanup_lockfile(self):
lf = self._env['lockfile']
if not self.isRunning() and os.path.exists(lf):
debug("Removing stale lock file for {} ...".format(
self._env['nick']))
os.remove(lf)
def verify(self, data):
# shortcut read when we don't ever expect any data
if self.repetitions == 0 or len(self.tt.data) == 0:
debug("no verification required - no data")
return 0
inp = self.s.recv(len(data) - len(self.inbuf))
debug("Verify: received %d bytes" % len(inp))
if len(inp) == 0:
debug("EOF on fd %s" % self.fd())
return -1
self.inbuf += inp
debug("successfully received (bytes=%d)" % len(self.inbuf))
while len(self.inbuf) >= len(data):
assert (len(self.inbuf) <= len(data) or self.repetitions > 1)
if self.inbuf[:len(data)] != data:
debug("receive comparison failed (bytes=%d)" % len(data))
return -1 # Failed verification.
# if we're not debugging, print a dot every dot_repetitions reps
elif (not debug_flag and self.tt.dot_repetitions > 0
and self.repetitions % self.tt.dot_repetitions == 0):
sys.stdout.write('.')
sys.stdout.flush()
# repeatedly check data against self.inbuf if required
debug("receive comparison success (bytes=%d)" % len(data))
self.inbuf = self.inbuf[len(data):]
debug("receive leftover bytes (bytes=%d)" % len(self.inbuf))
self.repetitions -= 1
debug("receive remaining repetitions (reps=%d)" % self.repetitions)
if self.repetitions == 0 and len(self.inbuf) == 0:
debug("successful verification")
# calculate the actual length of data remaining, including reps
debug("receive remaining bytes (bytes=%d)" %
(self.repetitions * len(data) - len(self.inbuf)))
return self.repetitions * len(data) - len(self.inbuf)
def accept(self):
newsock, endpoint = self.s.accept()
debug("new client from %s:%s (fd=%d)" %
(endpoint[0], endpoint[1], newsock.fileno()))
self.tt.add(Sink(self.tt, newsock))
def cleanup_lockfile(self):
lf = self._env['lockfile']
if not self.isRunning() and os.path.exists(lf):
debug("Removing stale lock file for {} ..."
.format(self._env['nick']))
os.remove(lf)
def accept(self):
newsock, endpoint = self.s.accept()
debug("new client from %s:%s (fd=%d)" %
(endpoint[0], endpoint[1], newsock.fileno()))
self.tt.add(Sink(self.tt, newsock))
def verify(self, data):
# shortcut read when we don't ever expect any data
if self.repetitions == 0 or len(self.tt.data) == 0:
debug("no verification required - no data")
return 0
inp = self.s.recv(len(data) - len(self.inbuf))
debug("Verify: received %d bytes"% len(inp))
if len(inp) == 0:
debug("EOF on fd %s" % self.fd())
return -1
self.inbuf += inp
debug("successfully received (bytes=%d)" % len(self.inbuf))
while len(self.inbuf) >= len(data):
assert(len(self.inbuf) <= len(data) or self.repetitions > 1)
if self.inbuf[:len(data)] != data:
debug("receive comparison failed (bytes=%d)" % len(data))
return -1 # Failed verification.
# if we're not debugging, print a dot every dot_repetitions reps
elif (not debug_flag and self.tt.dot_repetitions > 0 and
self.repetitions % self.tt.dot_repetitions == 0):
sys.stdout.write('.')
sys.stdout.flush()
# repeatedly check data against self.inbuf if required
debug("receive comparison success (bytes=%d)" % len(data))
self.inbuf = self.inbuf[len(data):]
debug("receive leftover bytes (bytes=%d)" % len(self.inbuf))
self.repetitions -= 1
debug("receive remaining repetitions (reps=%d)" % self.repetitions)
if self.repetitions == 0 and len(self.inbuf) == 0:
debug("successful verification")
# calculate the actual length of data remaining, including reps
debug("receive remaining bytes (bytes=%d)"
% (self.repetitions*len(data) - len(self.inbuf)))
return self.repetitions*len(data) - len(self.inbuf)
def on_writable(self):
"""Invoked when the socket becomes writable.
Return 0 when done writing
-1 on failure (like connection refused)
>0 if more data needs to be written
"""
if self.state == self.CONNECTING:
if self.proxy is None:
self.state = self.CONNECTED
debug("successfully connected (fd=%d)" % self.fd())
else:
self.state = self.CONNECTING_THROUGH_PROXY
self.outbuf = socks_cmd(self.dest)
# we write socks_cmd() to the proxy, then read the response
# if we get the correct response, we're CONNECTED
if self.state == self.CONNECTED:
# repeat self.data into self.outbuf if required
if (len(self.outbuf) < len(self.data) and self.repetitions > 0):
self.outbuf += self.data
self.repetitions -= 1
debug("adding more data to send (bytes=%d)" % len(self.data))
debug("now have data to send (bytes=%d)" % len(self.outbuf))
debug("send repetitions remaining (reps=%d)"
% self.repetitions)
try:
n = self.s.send(self.outbuf)
except socket.error as e:
if e.errno == errno.ECONNREFUSED:
debug("connection refused (fd=%d)" % self.fd())
return -1
raise
# sometimes, this debug statement prints 0
# it should print length of the data sent
# but the code works as long as this doesn't keep on happening
if n > 0:
debug("successfully sent (bytes=%d)" % n)
self._sent_no_bytes = 0
else:
debug("BUG: sent no bytes (out of %d; state is %s)"% (len(self.outbuf), self.state))
self._sent_no_bytes += 1
# We can't retry too fast, otherwise clients burn all their HSDirs
if self._sent_no_bytes >= 2:
print("Sent no data %d times. Stalled." %
(self._sent_no_bytes))
return -1
time.sleep(5)
self.outbuf = self.outbuf[n:]
if self.state == self.CONNECTING_THROUGH_PROXY:
return 1 # Keep us around.
debug("bytes remaining on outbuf (bytes=%d)" % len(self.outbuf))
# calculate the actual length of data remaining, including reps
# When 0, we're being removed.
debug("bytes remaining overall (bytes=%d)"
% (self.repetitions*len(self.data) + len(self.outbuf)))
return self.repetitions*len(self.data) + len(self.outbuf)
def run(self):
while not self.tests.all_done() and self.timeout > 0:
rset = [self.listener.fd()] + list(self.peers)
wset = [p.fd() for p in
filter(lambda x: x.want_to_write(), self.sources())]
# debug("rset %s wset %s" % (rset, wset))
sets = select.select(rset, wset, [], 1)
if all(len(s) == 0 for s in sets):
debug("Decrementing timeout.")
self.timeout -= 1
continue
for fd in sets[0]: # readable fd's
if fd == self.listener.fd():
self.listener.accept()
continue
p = self.peers[fd]
n = p.on_readable()
debug("On read, fd %d for %s said %d"%(fd, p, n))
if n > 0:
# debug("need %d more octets from fd %d" % (n, fd))
pass
elif n == 0: # Success.
self.tests.success()
self.remove(p)
else: # Failure.
debug("Got a failure reading fd %d for %s" % (fd,p))
self.tests.failure()
if p.is_sink():
print("verification failed!")
self.remove(p)
for fd in sets[1]: # writable fd's
p = self.peers.get(fd)
if p is not None: # Might have been removed above.
n = p.on_writable()
debug("On write, fd %d said %d"%(fd, n))
if n == 0:
self.remove(p)
elif n < 0:
debug("Got a failure writing fd %d for %s" % (fd,p))
self.tests.failure()
self.remove(p)
for fd in self.peers:
peer = self.peers[fd]
debug("peer fd=%d never pending close, never read or wrote" % fd)
self.pending_close.append(peer.s)
self.listener.s.close()
for s in self.pending_close:
s.close()
if not debug_flag:
sys.stdout.write('\n')
sys.stdout.flush()
debug("Done with run(); all_done == %s and failure_count == %s"
%(self.tests.all_done(), self.tests.failure_count()))
return self.tests.all_done() and self.tests.failure_count() == 0
def run(self):
while not self.tests.all_done() and self.timeout > 0:
rset = [self.listener.fd()] + list(self.peers)
wset = [
p.fd()
for p in filter(lambda x: x.want_to_write(), self.sources())
]
# debug("rset %s wset %s" % (rset, wset))
sets = select.select(rset, wset, [], 1)
if all(len(s) == 0 for s in sets):
debug("Decrementing timeout.")
self.timeout -= 1
continue
for fd in sets[0]: # readable fd's
if fd == self.listener.fd():
self.listener.accept()
continue
p = self.peers[fd]
n = p.on_readable()
debug("On read, fd %d for %s said %d" % (fd, p, n))
if n > 0:
# debug("need %d more octets from fd %d" % (n, fd))
pass
elif n == 0: # Success.
self.tests.success()
self.remove(p)
else: # Failure.
debug("Got a failure reading fd %d for %s" % (fd, p))
self.tests.failure()
if p.is_sink():
print("verification failed!")
self.remove(p)
for fd in sets[1]: # writable fd's
p = self.peers.get(fd)
if p is not None: # Might have been removed above.
n = p.on_writable()
debug("On write, fd %d said %d" % (fd, n))
if n == 0:
self.remove(p)
elif n < 0:
debug("Got a failure writing fd %d for %s" % (fd, p))
self.tests.failure()
self.remove(p)
for fd in self.peers:
peer = self.peers[fd]
debug("peer fd=%d never pending close, never read or wrote" % fd)
self.pending_close.append(peer.s)
self.listener.s.close()
for s in self.pending_close:
s.close()
if not debug_flag:
sys.stdout.write('\n')
sys.stdout.flush()
debug("Done with run(); all_done == %s and failure_count == %s" %
(self.tests.all_done(), self.tests.failure_count()))
return self.tests.all_done() and self.tests.failure_count() == 0
def on_readable(self):
"""Invoked when the socket becomes readable.
Return -1 on failure
>0 if more data needs to be read or written
"""
if self.state == self.CONNECTING_THROUGH_PROXY:
inp = self.s.recv(8 - len(self.inbuf))
debug("-- connecting through proxy, got %d bytes"%len(inp))
if len(inp) == 0:
debug("EOF on fd %d"%self.fd())
return -1
self.inbuf += inp
if len(self.inbuf) == 8:
if self.inbuf[:2] == b'\x00\x5a':
debug("proxy handshake successful (fd=%d)" % self.fd())
self.state = self.CONNECTED
self.inbuf = b''
debug("successfully connected (fd=%d)" % self.fd())
# if we have no reps or no data, skip sending actual data
if self.want_to_write():
return 1 # Keep us around for writing.
else:
# shortcut write when we don't ever expect any data
debug("no connection required - no data")
return 0
else:
debug("proxy handshake failed (0x%x)! (fd=%d)" %
(ord(self.inbuf[1]), self.fd()))
self.state = self.NOT_CONNECTED
return -1
assert(8 - len(self.inbuf) > 0)
return 8 - len(self.inbuf)
return self.want_to_write() # Keep us around for writing if needed
