def main(): ''' Unit test for neubot/utils_random.py ''' assert (len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz) assert (RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block()) filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0 while True: block = filep.read(128) if not block: break total += len(block) assert (total == RANDOMBLOCKS.blocksiz + 789) filep = RandomBody(RANDOMBLOCKS.blocksiz + 789) assert (len(filep.read()) == RANDOMBLOCKS.blocksiz) assert (filep.tell() == 789) assert (len(filep.read()) == 789) filep.seek(7) begin, total = utils.ticks(), 0 while total < 1073741824: total += len(RANDOMBLOCKS.get_block()) elapsed = utils.ticks() - begin print('Elapsed: %s' % utils.time_formatter(elapsed)) print('Speed: %s' % utils.speed_formatter(total / elapsed))
def connection_ready(self, stream): request = Message() # # With version 2, we upload bytes using chunked transfer # encoding for TARGET seconds. # if self.conf['version'] == 2: body = BytegenSpeedtest(TARGET) request.compose(method='POST', chunked=body, pathquery='/speedtest/upload', host=self.host_header) request['authorization'] = self.conf[ 'speedtest.client.authorization'] stream.send_request(request) self.ticks[stream] = utils.ticks() self.bytes[stream] = stream.bytes_sent_tot return request.compose(method="POST", body=RandomBody(ESTIMATE["upload"]), pathquery="/speedtest/upload", host=self.host_header) request["authorization"] = self.conf.get( "speedtest.client.authorization", "") self.ticks[stream] = utils.ticks() self.bytes[stream] = stream.bytes_sent_tot stream.send_request(request)
def main(): ''' Unit test for neubot/utils_random.py ''' assert(len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz) assert(RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block()) filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0 while True: block = filep.read(128) if not block: break total += len(block) assert(total == RANDOMBLOCKS.blocksiz + 789) filep = RandomBody(RANDOMBLOCKS.blocksiz + 789) assert(len(filep.read()) == RANDOMBLOCKS.blocksiz) assert(filep.tell() == 789) assert(len(filep.read()) == 789) filep.seek(7) begin, total = utils.ticks(), 0 while total < 1073741824: total += len(RANDOMBLOCKS.get_block()) elapsed = utils.ticks() - begin print('Elapsed: %s' % utils.time_formatter(elapsed)) print('Speed: %s' % utils.speed_formatter(total/elapsed))
def main(): assert(len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz) assert(RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block()) fp, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0 while True: block = fp.read(128) if not block: break total += len(block) assert(total == RANDOMBLOCKS.blocksiz + 789) fp = RandomBody(RANDOMBLOCKS.blocksiz + 789) assert(len(fp.read()) == RANDOMBLOCKS.blocksiz) assert(fp.tell() == 789) assert(len(fp.read()) == 789) fp.seek(7) begin, total = utils.ticks(), 0 try: while True: total += len(RANDOMBLOCKS.get_block()) except KeyboardInterrupt: print "Interrupt" print "Speed:", utils.speed_formatter(total/(utils.ticks() - begin))
def _waiting_pingback(self, stream, data): ''' Invoke when waiting for PINGBACK ''' #context = stream.opaque #context.bufferise(data) #tmp = context.pullup(len(PINGBACK)) #if not tmp: stream.recv(len(PINGBACK), self._waiting_pingback) return if tmp[4:5] != PINGBACK_CODE: raise RuntimeError('skype_clnt: received invalid message') timediff = utils.ticks() - context.alrtt_ticks context.state.setdefault('alrtt_list', []).append(timediff) logging.debug('< PINGBACK') logging.debug('skype_clnt: alrtt_sample: %f', timediff) context.alrtt_cnt -= 1 if context.alrtt_cnt > 0: self._send_ping(stream) return alrtt_list = context.state['alrtt_list'] alrtt_avg = sum(alrtt_list) / len(alrtt_list) context.state['alrtt_avg'] = alrtt_avg latency = utils.time_formatter(alrtt_avg) logging.info('skype_clnt: alrtt_avg: %s', latency) STATE.update('test_latency', latency) logging.info('skype_clnt: estimating ALRTT... complete') logging.info('skype_clnt: skype goodput test... in progress') logging.debug('> RAWTEST') stream.send(RAWTEST, self._skypetest_sent)
def resched(self, delta, *args, **kwargs): self.time = ticks() + delta self.timestamp = timestamp() + int(delta) if args: self.args = args if kwargs: self.kwargs = kwargs
def complete(self, done="done\n"): elapsed = utils.time_formatter(utils.ticks() - self.ticks) done = "".join([done.rstrip(), " [in ", elapsed, "]\n"]) if not self.message: self.message = "???" self.info(self.message + "..." + done) self.message = None
def _waiting_pingback(self, stream, data): """ Invoke when waiting for PINGBACK """ context = stream.opaque context.bufferise(data) tmp = context.pullup(len(PINGBACK)) if not tmp: stream.recv(len(PINGBACK), self._waiting_pingback) return if tmp[4:5] != PINGBACK_CODE: raise RuntimeError("raw_clnt: received invalid message") timediff = utils.ticks() - context.alrtt_ticks context.state.setdefault("alrtt_list", []).append(timediff) logging.debug("< PINGBACK") logging.debug("raw_clnt: alrtt_sample: %f", timediff) context.alrtt_cnt -= 1 if context.alrtt_cnt > 0: self._send_ping(stream) return alrtt_list = context.state["alrtt_list"] alrtt_avg = sum(alrtt_list) / len(alrtt_list) context.state["alrtt_avg"] = alrtt_avg latency = utils.time_formatter(alrtt_avg) logging.info("raw_clnt: alrtt_avg: %s", latency) STATE.update("test_progress", "50%", publish=False) STATE.update("test_latency", latency) logging.info("raw_clnt: estimating ALRTT... complete") logging.info("raw_clnt: raw goodput test... in progress") logging.debug("> RAWTEST") stream.send(RAWTEST, self._rawtest_sent)
def start_recv(self): if (self.close_complete or self.close_pending or self.recv_pending): return self.recv_ticks = utils.ticks() self.recv_pending = True if self.recv_blocked: return self.poller.set_readable(self) # # The client-side of an SSL connection must send the HELLO # message to start the negotiation. This is done automagically # by SSL_read and SSL_write. When the client first operation # is a send(), no problem: the socket is always writable at # the beginning and writable() invokes sosend() that invokes # SSL_write() that negotiates. The problem is when the client # first operation is recv(): in this case the socket would never # become readable because the server side is waiting for an HELLO. # The following piece of code makes sure that the first recv() # of the client invokes readable() that invokes sorecv() that # invokes SSL_read() that starts the negotiation. # if self.recv_ssl_needs_kickoff: self.recv_ssl_needs_kickoff = False self.handle_readable()
def start(self, message): self.ticks = utils.ticks() if self.noisy or not self.interactive: self.info(message + " in progress...") self.message = message else: sys.stderr.write(message + "...")
def check_timeout(self, *args, **kwargs): self.sched(CHECK_TIMEOUT, self.check_timeout) if self.readset or self.writeset: now = ticks() stale = set() x = self.readset.values() for stream in x: if stream.readtimeout(now): logging.warning("%s: read timeout" % repr(stream)) stale.add(stream) elif (stream.watchdog > 0 and now - stream.created > stream.watchdog): logging.warning("%s: watchdog timeout" % repr(stream)) stale.add(stream) x = self.writeset.values() for stream in x: if stream.writetimeout(now): logging.warning("%s: write timeout" % repr(stream)) stale.add(stream) elif (stream.watchdog > 0 and now - stream.created > stream.watchdog): logging.warning("%s: watchdog timeout" % repr(stream)) stale.add(stream) for stream in stale: self.close(stream)
def test_negotiate_add(self): """Verify negotiate behavior when we add a new stream""" stream = Stream(None) stream.peername = ("abc", 3456) m = { "stream": stream, "ident": sha1stream(stream), "module": "abc", } negotiator = _Negotiator() negotiator._finalize_response = lambda m, ln: None negotiator._send_response = lambda m: None negotiator.register("abc", None) negotiator.negotiate(m) # Do we keep track of the new stream? self.assertTrue(stream in negotiator._queue) self.assertTrue(stream in negotiator._known) # Is the watchdog correctly initialized? self.assertTrue(negotiator._at_close in stream.atclosev) self.assertEqual(stream.watchdog, 300) self.assertTrue(utils.ticks() - stream.created < 1) #XXX
def got_piece(self, *args): stream = args[0] if self.state != DOWNLOADING: raise RuntimeError("PIECE when state != DOWNLOADING") # Start measuring if not self.saved_ticks: self.saved_bytes = stream.bytes_recv_tot self.saved_ticks = utils.ticks() # # The download is driven by the sender and # we just need to discard the pieces. # Periodically send some requests to the other # end, with probability 10%. # if self.version >= 2: if self.version == 3 or random.random() < 0.1: index = random.randrange(self.numpieces) stream.send_request(index, 0, PIECE_LEN) return self.get_piece_old(stream)
def _periodic_internal(stream): """ Periodically snap goodput (internal function) """ context = stream.opaque utime, stime = os.times()[:2] utimediff = utime - context.snap_utime stimediff = stime - context.snap_stime ticks = utils.ticks() timediff = ticks - context.snap_ticks bytesdiff = stream.bytes_in - context.snap_count context.state.setdefault("goodput_snap", []).append( { "ticks": ticks, "bytesdiff": bytesdiff, "timediff": timediff, "utimediff": utimediff, "stimediff": stimediff, } ) logging.debug("raw_clnt: utime, stime = %f, %f", utime, stime) if timediff > 1e-06: speed = utils.speed_formatter(bytesdiff / timediff) logging.debug("raw_clnt: goodput_snap: %s", speed) context.snap_count = stream.bytes_in context.snap_ticks = ticks context.snap_utime = utime context.snap_stime = stime
def send_complete(self, stream): ''' Invoked when the send queue is empty ''' if self.version >= 2 and self.state == UPLOADING: # # The sender stops sending when the upload has run for # enough time, notifies peer with CHOKE and waits for # NOT_INTERESTED. # diff = utils.ticks() - self.begin_upload if diff > TARGET: self.state = WAIT_NOT_INTERESTED stream.send_choke() return if self.version == 3: return # # TODO Here we should use the random block # generator but before we do that we should # also make sure it does not slow down the # bittorrent test. # block = chr(random.randint(32, 126)) * PIECE_LEN index = random.randrange(self.numpieces) stream.send_piece(index, 0, block)
def _waiting_rawtest(self, stream, data): ''' Waiting for RAWTEST message from client ''' context = stream.opaque context.bufferise(data) tmp = context.pullup(len(RAWTEST)) if not tmp: stream.recv(len(RAWTEST), self._waiting_rawtest) return if tmp[4:5] == PING_CODE: logging.debug('< PING') stream.send(PINGBACK, self._sent_pingback) logging.debug('> PINGBACK') return if tmp[4:5] != RAWTEST_CODE: raise RuntimeError('raw_srvr: received invalid message') logging.debug('< RAWTEST') logging.info('raw_srvr: waiting for RAWTEST message... complete') logging.info('raw_srvr: raw test... in progress') context.count = context.snap_count = stream.bytes_out context.ticks = context.snap_ticks = utils.ticks() context.snap_utime, context.snap_stime = os.times()[:2] message = PIECE_CODE + context.auth * int(LEN_MESSAGE / AUTH_LEN) context.message = struct.pack('!I', len(message)) + message stream.send(context.message, self._piece_sent) #logging.debug('> PIECE') POLLER.sched(1, self._periodic, stream) stream.recv(1, self._waiting_eof)
def _waiting_piece(self, stream, data): ''' Invoked when new data is available ''' # Note: this loop cannot be adapted to process other messages # easily, as pointed out in <skype_defs.py>. context = stream.opaque context.bufferise(data) context.state['rcvr_data'].append((utils.ticks(), len(data))) while True: if context.left > 0: context.left = context.skip(context.left) if context.left > 0: break elif context.left == 0: tmp = context.pullup(4) if not tmp: break context.left, = struct.unpack('!I', tmp) if context.left > MAXRECV: raise RuntimeError('skype_clnt: PIECE too large') if not context.ticks: context.ticks = context.snap_ticks = utils.ticks() context.count = context.snap_count = stream.bytes_in context.snap_utime, context.snap_stime = os.times()[:2] POLLER.sched(1, self._periodic, stream) if context.left == 0: logging.debug('< {empty-message}') logging.info('skype_clnt: skype goodput test... complete') ticks = utils.ticks() timediff = ticks - context.ticks bytesdiff = stream.bytes_in - context.count context.state['goodput'] = { 'ticks': ticks, 'bytesdiff': bytesdiff, 'timediff': timediff, } if timediff > 1e-06: speed = utils.speed_formatter(bytesdiff / timediff) logging.info('skype_clnt: goodput: %s', speed) STATE.update('test_download', speed, publish=0) STATE.update('test_upload', 'N/A') self._periodic_internal(stream) context.state['complete'] = 1 stream.close() return else: raise RuntimeError('skype_clnt: internal error') stream.recv(MAXRECV, self._waiting_piece)
def connection_ready(self, stream): request = Message() request.compose(method="HEAD", pathquery="/speedtest/latency", host=self.host_header) request["authorization"] = self.conf.get( "speedtest.client.authorization", "") self.ticks[stream] = utils.ticks() stream.send_request(request)
def getaddrinfo(address, port, family=0, socktype=0, proto=0, flags=0): now = utils.ticks() if address in DNS_CACHE: addrinfo, ticks = DNS_CACHE[(address, port)] if now - ticks <= DNS_TIMEOUT: return addrinfo addrinfo = socket.getaddrinfo(address, port, family, socktype, proto, flags) DNS_CACHE[(address, port)] = (addrinfo, now) return addrinfo
def connection_ready(self, stream): request = Message() request.compose(method="POST", body=RandomBody(ESTIMATE["upload"]), pathquery="/speedtest/upload", host=self.host_header) request["authorization"] = self.conf.get( "speedtest.client.authorization", "") self.ticks[stream] = utils.ticks() self.bytes[stream] = stream.bytes_sent_tot stream.send_request(request)
def handle_write(self): self.poller.unset_writable(self) if not utils_net.isconnected(self.endpoint, self.sock): self._connection_failed() return rtt = utils.ticks() - self.timestamp self.parent._connection_made(self.sock, self.endpoint, rtt)
def _connect(self): ''' Connect first available epnt ''' sock = utils_net.connect(self.epnts.popleft(), self.prefer_ipv6) if sock: self.sock = sock self.timestamp = utils.ticks() self.poller.set_writable(self) else: self._connection_failed()
def handle_write(self): self.poller.unset_writable(self) if not utils_net.isconnected(self.endpoint, self.sock): self._connection_failed() return deferred = Deferred() deferred.add_callback(self._handle_connect) deferred.add_errback(self._handle_connect_error) deferred.callback(utils.ticks())
def _waiting_piece(self, stream, data): """ Invoked when new data is available """ # Note: this loop cannot be adapted to process other messages # easily, as pointed out in <raw_defs.py>. context = stream.opaque context.bufferise(data) context.state["rcvr_data"].append((utils.ticks(), len(data))) while True: if context.left > 0: context.left = context.skip(context.left) if context.left > 0: break elif context.left == 0: tmp = context.pullup(4) if not tmp: break context.left, = struct.unpack("!I", tmp) if context.left > MAXRECV: raise RuntimeError("raw_clnt: PIECE too large") if not context.ticks: context.ticks = context.snap_ticks = utils.ticks() context.count = context.snap_count = stream.bytes_in context.snap_utime, context.snap_stime = os.times()[:2] POLLER.sched(1, self._periodic, stream) if context.left == 0: logging.debug("< {empty-message}") logging.info("raw_clnt: raw goodput test... complete") ticks = utils.ticks() timediff = ticks - context.ticks bytesdiff = stream.bytes_in - context.count context.state["goodput"] = {"ticks": ticks, "bytesdiff": bytesdiff, "timediff": timediff} if timediff > 1e-06: speed = utils.speed_formatter(bytesdiff / timediff) logging.info("raw_clnt: goodput: %s", speed) STATE.update("test_progress", "100%", publish=False) STATE.update("test_download", speed, publish=0) STATE.update("test_upload", "N/A") self._periodic_internal(stream) context.state["complete"] = 1 stream.close() return else: raise RuntimeError("raw_clnt: internal error") stream.recv(MAXRECV, self._waiting_piece)
def complete(self, done="done\n"): elapsed = utils.time_formatter(utils.ticks() - self.ticks) done = "".join([done.rstrip(), " [in ", elapsed, "]\n"]) if self.noisy or not self.interactive: if not self.message: self.message = "???" self.info(self.message + "..." + done) self.message = None else: sys.stderr.write(done)
def connection_ready(self, stream): request = Message() request.compose(method="GET", pathquery="/speedtest/download", host=self.host_header) request["range"] = "bytes=0-%d" % ESTIMATE['download'] request["authorization"] = self.conf.get( "speedtest.client.authorization", "") self.ticks[stream] = utils.ticks() self.bytes[stream] = stream.bytes_recv_tot response = Message() response.body.write = lambda piece: None stream.send_request(request, response)
def process_request(self, stream, request): stream.created = utils.ticks() try: self._serve_request(stream, request) except ConfigError, error: reason = re.sub(r"[\0-\31]", "", str(error)) reason = re.sub(r"[\x7f-\xff]", "", reason) LOG.exception(func=LOG.info) response = Message() response.compose(code="500", reason=reason, body=StringIO.StringIO(reason)) stream.send_response(request, response)
def check_timeout(self): self.sched(CHECK_TIMEOUT, self.check_timeout) if self.readset or self.writeset: streams = set() streams.update(self.readset.values()) streams.update(self.writeset.values()) timenow = ticks() for stream in streams: if stream.handle_periodic(timenow): logging.warning('Watchdog timeout: %s', str(stream)) self.close(stream)
def process_request(self, stream, request): ''' Process incoming request ''' stream.created = utils.ticks() try: self._serve_request(stream, request) except (ConfigError, NotImplementedTest): error = sys.exc_info()[1] reason = re.sub(r"[\0-\31]", "", str(error)) reason = re.sub(r"[\x7f-\xff]", "", reason) logging.info('Internal error while serving response', exc_info=1) response = Message() response.compose(code="500", reason=reason, body=reason) stream.send_response(request, response)
def check_timeout(self): ''' Dispatch the periodic event ''' self.sched(CHECK_TIMEOUT, self.check_timeout) if self.readset or self.writeset: streams = set() streams.update(list(self.readset.values())) streams.update(list(self.writeset.values())) timenow = ticks() for stream in streams: if stream.handle_periodic(timenow): logging.debug('poller: watchdog timeout: %s', str(stream)) self.close(stream)
def _piece_sent(self, stream): ''' Invoked when a message has been sent ''' context = stream.opaque ticks = utils.ticks() if ticks - context.ticks < 10: stream.send(context.message, self._piece_sent) #logging.debug('> PIECE') return logging.info('raw_srvr: raw test... complete') ticks = utils.ticks() timediff = ticks - context.ticks bytesdiff = stream.bytes_out - context.count context.state['timestamp'] = utils.timestamp() context.state['goodput'] = { 'ticks': ticks, 'bytesdiff': bytesdiff, 'timediff': timediff, } if timediff > 1e-06: speed = utils.speed_formatter(bytesdiff / timediff) logging.info('raw_srvr: goodput: %s', speed) self._periodic_internal(stream) stream.send(EMPTY_MESSAGE, self._empty_message_sent) logging.debug('> {empty-message}')
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Neubot. If not, see <http://www.gnu.org/licenses/>. # ''' Unit test for neubot/utils_random.py ''' import sys sys.path.insert(0, '.') from neubot import utils BEFORE = utils.ticks() from neubot.utils_random import RANDOMBLOCKS from neubot.utils_random import RandomBody ELAPSED = utils.ticks() - BEFORE print('Time to import: %s' % (utils.time_formatter(ELAPSED))) def main(): ''' Unit test for neubot/utils_random.py ''' assert (len(RANDOMBLOCKS.get_block()) == RANDOMBLOCKS.blocksiz) assert (RANDOMBLOCKS.get_block() != RANDOMBLOCKS.get_block()) filep, total = RandomBody(RANDOMBLOCKS.blocksiz + 789), 0 while True: block = filep.read(128)