def main():
''' Unit test for neubot/utils_random.py '''
assert (len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz)
assert (RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block())
filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0
while True:
block = filep.read(128)
if not block:
break
total += len(block)
assert (total == RANDOMBLOCKS.blocksiz + 789)
filep = RandomBody(RANDOMBLOCKS.blocksiz + 789)
assert (len(filep.read()) == RANDOMBLOCKS.blocksiz)
assert (filep.tell() == 789)
assert (len(filep.read()) == 789)
filep.seek(7)
begin, total = utils.ticks(), 0
while total < 1073741824:
total += len(RANDOMBLOCKS.get_block())
elapsed = utils.ticks() - begin
print('Elapsed: %s' % utils.time_formatter(elapsed))
print('Speed: %s' % utils.speed_formatter(total / elapsed))
def connection_ready(self, stream):
request = Message()
#
# With version 2, we upload bytes using chunked transfer
# encoding for TARGET seconds.
#
if self.conf['version'] == 2:
body = BytegenSpeedtest(TARGET)
request.compose(method='POST', chunked=body,
pathquery='/speedtest/upload', host=self.host_header)
request['authorization'] = self.conf[
'speedtest.client.authorization']
stream.send_request(request)
self.ticks[stream] = utils.ticks()
self.bytes[stream] = stream.bytes_sent_tot
return
request.compose(method="POST", body=RandomBody(ESTIMATE["upload"]),
pathquery="/speedtest/upload", host=self.host_header)
request["authorization"] = self.conf.get(
"speedtest.client.authorization", "")
self.ticks[stream] = utils.ticks()
self.bytes[stream] = stream.bytes_sent_tot
stream.send_request(request)
def main():
''' Unit test for neubot/utils_random.py '''
assert(len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz)
assert(RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block())
filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0
while True:
block = filep.read(128)
if not block:
break
total += len(block)
assert(total == RANDOMBLOCKS.blocksiz + 789)
filep = RandomBody(RANDOMBLOCKS.blocksiz + 789)
assert(len(filep.read()) == RANDOMBLOCKS.blocksiz)
assert(filep.tell() == 789)
assert(len(filep.read()) == 789)
filep.seek(7)
begin, total = utils.ticks(), 0
while total < 1073741824:
total += len(RANDOMBLOCKS.get_block())
elapsed = utils.ticks() - begin
print('Elapsed: %s' % utils.time_formatter(elapsed))
print('Speed: %s' % utils.speed_formatter(total/elapsed))
def main():
assert(len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz)
assert(RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block())
fp, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0
while True:
block = fp.read(128)
if not block:
break
total += len(block)
assert(total == RANDOMBLOCKS.blocksiz + 789)
fp = RandomBody(RANDOMBLOCKS.blocksiz + 789)
assert(len(fp.read()) == RANDOMBLOCKS.blocksiz)
assert(fp.tell() == 789)
assert(len(fp.read()) == 789)
fp.seek(7)
begin, total = utils.ticks(), 0
try:
while True:
total += len(RANDOMBLOCKS.get_block())
except KeyboardInterrupt:
print "Interrupt"
print "Speed:", utils.speed_formatter(total/(utils.ticks() - begin))
def _waiting_pingback(self, stream, data):
''' Invoke when waiting for PINGBACK '''
#context = stream.opaque
#context.bufferise(data)
#tmp = context.pullup(len(PINGBACK))
#if not tmp:
stream.recv(len(PINGBACK), self._waiting_pingback)
return
if tmp[4:5] != PINGBACK_CODE:
raise RuntimeError('skype_clnt: received invalid message')
timediff = utils.ticks() - context.alrtt_ticks
context.state.setdefault('alrtt_list', []).append(timediff)
logging.debug('< PINGBACK')
logging.debug('skype_clnt: alrtt_sample: %f', timediff)
context.alrtt_cnt -= 1
if context.alrtt_cnt > 0:
self._send_ping(stream)
return
alrtt_list = context.state['alrtt_list']
alrtt_avg = sum(alrtt_list) / len(alrtt_list)
context.state['alrtt_avg'] = alrtt_avg
latency = utils.time_formatter(alrtt_avg)
logging.info('skype_clnt: alrtt_avg: %s', latency)
STATE.update('test_latency', latency)
logging.info('skype_clnt: estimating ALRTT... complete')
logging.info('skype_clnt: skype goodput test... in progress')
logging.debug('> RAWTEST')
stream.send(RAWTEST, self._skypetest_sent)
def resched(self, delta, *args, **kwargs):
self.time = ticks() + delta
self.timestamp = timestamp() + int(delta)
if args:
self.args = args
if kwargs:
self.kwargs = kwargs
def complete(self, done="done\n"):
elapsed = utils.time_formatter(utils.ticks() - self.ticks)
done = "".join([done.rstrip(), " [in ", elapsed, "]\n"])
if not self.message:
self.message = "???"
self.info(self.message + "..." + done)
self.message = None
def _waiting_pingback(self, stream, data):
""" Invoke when waiting for PINGBACK """
context = stream.opaque
context.bufferise(data)
tmp = context.pullup(len(PINGBACK))
if not tmp:
stream.recv(len(PINGBACK), self._waiting_pingback)
return
if tmp[4:5] != PINGBACK_CODE:
raise RuntimeError("raw_clnt: received invalid message")
timediff = utils.ticks() - context.alrtt_ticks
context.state.setdefault("alrtt_list", []).append(timediff)
logging.debug("< PINGBACK")
logging.debug("raw_clnt: alrtt_sample: %f", timediff)
context.alrtt_cnt -= 1
if context.alrtt_cnt > 0:
self._send_ping(stream)
return
alrtt_list = context.state["alrtt_list"]
alrtt_avg = sum(alrtt_list) / len(alrtt_list)
context.state["alrtt_avg"] = alrtt_avg
latency = utils.time_formatter(alrtt_avg)
logging.info("raw_clnt: alrtt_avg: %s", latency)
STATE.update("test_progress", "50%", publish=False)
STATE.update("test_latency", latency)
logging.info("raw_clnt: estimating ALRTT... complete")
logging.info("raw_clnt: raw goodput test... in progress")
logging.debug("> RAWTEST")
stream.send(RAWTEST, self._rawtest_sent)
def start_recv(self):
if (self.close_complete or self.close_pending
or self.recv_pending):
return
self.recv_ticks = utils.ticks()
self.recv_pending = True
if self.recv_blocked:
return
self.poller.set_readable(self)
#
# The client-side of an SSL connection must send the HELLO
# message to start the negotiation. This is done automagically
# by SSL_read and SSL_write. When the client first operation
# is a send(), no problem: the socket is always writable at
# the beginning and writable() invokes sosend() that invokes
# SSL_write() that negotiates. The problem is when the client
# first operation is recv(): in this case the socket would never
# become readable because the server side is waiting for an HELLO.
# The following piece of code makes sure that the first recv()
# of the client invokes readable() that invokes sorecv() that
# invokes SSL_read() that starts the negotiation.
#
if self.recv_ssl_needs_kickoff:
self.recv_ssl_needs_kickoff = False
self.handle_readable()
def start(self, message):
self.ticks = utils.ticks()
if self.noisy or not self.interactive:
self.info(message + " in progress...")
self.message = message
else:
sys.stderr.write(message + "...")
def check_timeout(self, *args, **kwargs):
self.sched(CHECK_TIMEOUT, self.check_timeout)
if self.readset or self.writeset:
now = ticks()
stale = set()
x = self.readset.values()
for stream in x:
if stream.readtimeout(now):
logging.warning("%s: read timeout" % repr(stream))
stale.add(stream)
elif (stream.watchdog > 0 and
now - stream.created > stream.watchdog):
logging.warning("%s: watchdog timeout" % repr(stream))
stale.add(stream)
x = self.writeset.values()
for stream in x:
if stream.writetimeout(now):
logging.warning("%s: write timeout" % repr(stream))
stale.add(stream)
elif (stream.watchdog > 0 and
now - stream.created > stream.watchdog):
logging.warning("%s: watchdog timeout" % repr(stream))
stale.add(stream)
for stream in stale:
self.close(stream)
def test_negotiate_add(self):
"""Verify negotiate behavior when we add a new stream"""
stream = Stream(None)
stream.peername = ("abc", 3456)
m = {
"stream": stream,
"ident": sha1stream(stream),
"module": "abc",
}
negotiator = _Negotiator()
negotiator._finalize_response = lambda m, ln: None
negotiator._send_response = lambda m: None
negotiator.register("abc", None)
negotiator.negotiate(m)
# Do we keep track of the new stream?
self.assertTrue(stream in negotiator._queue)
self.assertTrue(stream in negotiator._known)
# Is the watchdog correctly initialized?
self.assertTrue(negotiator._at_close in stream.atclosev)
self.assertEqual(stream.watchdog, 300)
self.assertTrue(utils.ticks() - stream.created < 1) #XXX
def got_piece(self, *args):
stream = args[0]
if self.state != DOWNLOADING:
raise RuntimeError("PIECE when state != DOWNLOADING")
# Start measuring
if not self.saved_ticks:
self.saved_bytes = stream.bytes_recv_tot
self.saved_ticks = utils.ticks()
#
# The download is driven by the sender and
# we just need to discard the pieces.
# Periodically send some requests to the other
# end, with probability 10%.
#
if self.version >= 2:
if self.version == 3 or random.random() < 0.1:
index = random.randrange(self.numpieces)
stream.send_request(index, 0, PIECE_LEN)
return
self.get_piece_old(stream)
def _periodic_internal(stream):
""" Periodically snap goodput (internal function) """
context = stream.opaque
utime, stime = os.times()[:2]
utimediff = utime - context.snap_utime
stimediff = stime - context.snap_stime
ticks = utils.ticks()
timediff = ticks - context.snap_ticks
bytesdiff = stream.bytes_in - context.snap_count
context.state.setdefault("goodput_snap", []).append(
{
"ticks": ticks,
"bytesdiff": bytesdiff,
"timediff": timediff,
"utimediff": utimediff,
"stimediff": stimediff,
}
)
logging.debug("raw_clnt: utime, stime = %f, %f", utime, stime)
if timediff > 1e-06:
speed = utils.speed_formatter(bytesdiff / timediff)
logging.debug("raw_clnt: goodput_snap: %s", speed)
context.snap_count = stream.bytes_in
context.snap_ticks = ticks
context.snap_utime = utime
context.snap_stime = stime
def send_complete(self, stream):
''' Invoked when the send queue is empty '''
if self.version >= 2 and self.state == UPLOADING:
#
# The sender stops sending when the upload has run for
# enough time, notifies peer with CHOKE and waits for
# NOT_INTERESTED.
#
diff = utils.ticks() - self.begin_upload
if diff > TARGET:
self.state = WAIT_NOT_INTERESTED
stream.send_choke()
return
if self.version == 3:
return
#
# TODO Here we should use the random block
# generator but before we do that we should
# also make sure it does not slow down the
# bittorrent test.
#
block = chr(random.randint(32, 126)) * PIECE_LEN
index = random.randrange(self.numpieces)
stream.send_piece(index, 0, block)
def _waiting_rawtest(self, stream, data):
''' Waiting for RAWTEST message from client '''
context = stream.opaque
context.bufferise(data)
tmp = context.pullup(len(RAWTEST))
if not tmp:
stream.recv(len(RAWTEST), self._waiting_rawtest)
return
if tmp[4:5] == PING_CODE:
logging.debug('< PING')
stream.send(PINGBACK, self._sent_pingback)
logging.debug('> PINGBACK')
return
if tmp[4:5] != RAWTEST_CODE:
raise RuntimeError('raw_srvr: received invalid message')
logging.debug('< RAWTEST')
logging.info('raw_srvr: waiting for RAWTEST message... complete')
logging.info('raw_srvr: raw test... in progress')
context.count = context.snap_count = stream.bytes_out
context.ticks = context.snap_ticks = utils.ticks()
context.snap_utime, context.snap_stime = os.times()[:2]
message = PIECE_CODE + context.auth * int(LEN_MESSAGE / AUTH_LEN)
context.message = struct.pack('!I', len(message)) + message
stream.send(context.message, self._piece_sent)
#logging.debug('> PIECE')
POLLER.sched(1, self._periodic, stream)
stream.recv(1, self._waiting_eof)
def _waiting_piece(self, stream, data):
''' Invoked when new data is available '''
# Note: this loop cannot be adapted to process other messages
# easily, as pointed out in <skype_defs.py>.
context = stream.opaque
context.bufferise(data)
context.state['rcvr_data'].append((utils.ticks(), len(data)))
while True:
if context.left > 0:
context.left = context.skip(context.left)
if context.left > 0:
break
elif context.left == 0:
tmp = context.pullup(4)
if not tmp:
break
context.left, = struct.unpack('!I', tmp)
if context.left > MAXRECV:
raise RuntimeError('skype_clnt: PIECE too large')
if not context.ticks:
context.ticks = context.snap_ticks = utils.ticks()
context.count = context.snap_count = stream.bytes_in
context.snap_utime, context.snap_stime = os.times()[:2]
POLLER.sched(1, self._periodic, stream)
if context.left == 0:
logging.debug('< {empty-message}')
logging.info('skype_clnt: skype goodput test... complete')
ticks = utils.ticks()
timediff = ticks - context.ticks
bytesdiff = stream.bytes_in - context.count
context.state['goodput'] = {
'ticks': ticks,
'bytesdiff': bytesdiff,
'timediff': timediff,
}
if timediff > 1e-06:
speed = utils.speed_formatter(bytesdiff / timediff)
logging.info('skype_clnt: goodput: %s', speed)
STATE.update('test_download', speed, publish=0)
STATE.update('test_upload', 'N/A')
self._periodic_internal(stream)
context.state['complete'] = 1
stream.close()
return
else:
raise RuntimeError('skype_clnt: internal error')
stream.recv(MAXRECV, self._waiting_piece)
def connection_ready(self, stream):
request = Message()
request.compose(method="HEAD", pathquery="/speedtest/latency",
host=self.host_header)
request["authorization"] = self.conf.get(
"speedtest.client.authorization", "")
self.ticks[stream] = utils.ticks()
stream.send_request(request)
def getaddrinfo(address, port, family=0, socktype=0, proto=0, flags=0):
now = utils.ticks()
if address in DNS_CACHE:
addrinfo, ticks = DNS_CACHE[(address, port)]
if now - ticks <= DNS_TIMEOUT:
return addrinfo
addrinfo = socket.getaddrinfo(address, port, family, socktype, proto, flags)
DNS_CACHE[(address, port)] = (addrinfo, now)
return addrinfo
def connection_ready(self, stream):
request = Message()
request.compose(method="POST", body=RandomBody(ESTIMATE["upload"]),
pathquery="/speedtest/upload", host=self.host_header)
request["authorization"] = self.conf.get(
"speedtest.client.authorization", "")
self.ticks[stream] = utils.ticks()
self.bytes[stream] = stream.bytes_sent_tot
stream.send_request(request)
def handle_write(self):
self.poller.unset_writable(self)
if not utils_net.isconnected(self.endpoint, self.sock):
self._connection_failed()
return
rtt = utils.ticks() - self.timestamp
self.parent._connection_made(self.sock, self.endpoint, rtt)
def _connect(self):
''' Connect first available epnt '''
sock = utils_net.connect(self.epnts.popleft(), self.prefer_ipv6)
if sock:
self.sock = sock
self.timestamp = utils.ticks()
self.poller.set_writable(self)
else:
self._connection_failed()
def handle_write(self):
self.poller.unset_writable(self)
if not utils_net.isconnected(self.endpoint, self.sock):
self._connection_failed()
return
deferred = Deferred()
deferred.add_callback(self._handle_connect)
deferred.add_errback(self._handle_connect_error)
deferred.callback(utils.ticks())
def _waiting_piece(self, stream, data):
""" Invoked when new data is available """
# Note: this loop cannot be adapted to process other messages
# easily, as pointed out in <raw_defs.py>.
context = stream.opaque
context.bufferise(data)
context.state["rcvr_data"].append((utils.ticks(), len(data)))
while True:
if context.left > 0:
context.left = context.skip(context.left)
if context.left > 0:
break
elif context.left == 0:
tmp = context.pullup(4)
if not tmp:
break
context.left, = struct.unpack("!I", tmp)
if context.left > MAXRECV:
raise RuntimeError("raw_clnt: PIECE too large")
if not context.ticks:
context.ticks = context.snap_ticks = utils.ticks()
context.count = context.snap_count = stream.bytes_in
context.snap_utime, context.snap_stime = os.times()[:2]
POLLER.sched(1, self._periodic, stream)
if context.left == 0:
logging.debug("< {empty-message}")
logging.info("raw_clnt: raw goodput test... complete")
ticks = utils.ticks()
timediff = ticks - context.ticks
bytesdiff = stream.bytes_in - context.count
context.state["goodput"] = {"ticks": ticks, "bytesdiff": bytesdiff, "timediff": timediff}
if timediff > 1e-06:
speed = utils.speed_formatter(bytesdiff / timediff)
logging.info("raw_clnt: goodput: %s", speed)
STATE.update("test_progress", "100%", publish=False)
STATE.update("test_download", speed, publish=0)
STATE.update("test_upload", "N/A")
self._periodic_internal(stream)
context.state["complete"] = 1
stream.close()
return
else:
raise RuntimeError("raw_clnt: internal error")
stream.recv(MAXRECV, self._waiting_piece)
def complete(self, done="done\n"):
elapsed = utils.time_formatter(utils.ticks() - self.ticks)
done = "".join([done.rstrip(), " [in ", elapsed, "]\n"])
if self.noisy or not self.interactive:
if not self.message:
self.message = "???"
self.info(self.message + "..." + done)
self.message = None
else:
sys.stderr.write(done)
def connection_ready(self, stream):
request = Message()
request.compose(method="GET", pathquery="/speedtest/download",
host=self.host_header)
request["range"] = "bytes=0-%d" % ESTIMATE['download']
request["authorization"] = self.conf.get(
"speedtest.client.authorization", "")
self.ticks[stream] = utils.ticks()
self.bytes[stream] = stream.bytes_recv_tot
response = Message()
response.body.write = lambda piece: None
stream.send_request(request, response)
def process_request(self, stream, request):
stream.created = utils.ticks()
try:
self._serve_request(stream, request)
except ConfigError, error:
reason = re.sub(r"[\0-\31]", "", str(error))
reason = re.sub(r"[\x7f-\xff]", "", reason)
LOG.exception(func=LOG.info)
response = Message()
response.compose(code="500", reason=reason,
body=StringIO.StringIO(reason))
stream.send_response(request, response)
def check_timeout(self):
self.sched(CHECK_TIMEOUT, self.check_timeout)
if self.readset or self.writeset:
streams = set()
streams.update(self.readset.values())
streams.update(self.writeset.values())
timenow = ticks()
for stream in streams:
if stream.handle_periodic(timenow):
logging.warning('Watchdog timeout: %s', str(stream))
self.close(stream)
def process_request(self, stream, request):
''' Process incoming request '''
stream.created = utils.ticks()
try:
self._serve_request(stream, request)
except (ConfigError, NotImplementedTest):
error = sys.exc_info()[1]
reason = re.sub(r"[\0-\31]", "", str(error))
reason = re.sub(r"[\x7f-\xff]", "", reason)
logging.info('Internal error while serving response', exc_info=1)
response = Message()
response.compose(code="500", reason=reason,
body=reason)
stream.send_response(request, response)
def check_timeout(self):
''' Dispatch the periodic event '''
self.sched(CHECK_TIMEOUT, self.check_timeout)
if self.readset or self.writeset:
streams = set()
streams.update(list(self.readset.values()))
streams.update(list(self.writeset.values()))
timenow = ticks()
for stream in streams:
if stream.handle_periodic(timenow):
logging.debug('poller: watchdog timeout: %s', str(stream))
self.close(stream)
def _piece_sent(self, stream):
''' Invoked when a message has been sent '''
context = stream.opaque
ticks = utils.ticks()
if ticks - context.ticks < 10:
stream.send(context.message, self._piece_sent)
#logging.debug('> PIECE')
return
logging.info('raw_srvr: raw test... complete')
ticks = utils.ticks()
timediff = ticks - context.ticks
bytesdiff = stream.bytes_out - context.count
context.state['timestamp'] = utils.timestamp()
context.state['goodput'] = {
'ticks': ticks,
'bytesdiff': bytesdiff,
'timediff': timediff,
}
if timediff > 1e-06:
speed = utils.speed_formatter(bytesdiff / timediff)
logging.info('raw_srvr: goodput: %s', speed)
self._periodic_internal(stream)
stream.send(EMPTY_MESSAGE, self._empty_message_sent)
logging.debug('> {empty-message}')
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Neubot. If not, see <http://www.gnu.org/licenses/>.
#
''' Unit test for neubot/utils_random.py '''
import sys
sys.path.insert(0, '.')
from neubot import utils
BEFORE = utils.ticks()
from neubot.utils_random import RANDOMBLOCKS
from neubot.utils_random import RandomBody
ELAPSED = utils.ticks() - BEFORE
print('Time to import: %s' % (utils.time_formatter(ELAPSED)))
def main():
''' Unit test for neubot/utils_random.py '''
assert (len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz)
assert (RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block())
filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0
while True:
block = filep.read(128)
