def test_different_delay(self):
# NOTICE: this test might fail if your configuration
# (interpreter/processor) is too slow
task_delays = (1, 1, 1, .5, 1, 1, 2, 1, 1, 1,
1, 1.5, 1, 1, 1, 1, .3)
expected_list = ([],
['a', 16, 3, 'b'], #9 is cancelled
['a', 0, 1, 2, 4, 5, 7, 8, 10, 12, 13, 15, 'c', 'b'],
['a', 11, 'c', 'b'],
['a', 6, 'c', 'b'],
)
tasks = [Task(delay, self.callback_f, i) \
for i, delay in enumerate(task_delays)]
for task in tasks:
self.task_m.add(task)
for i, expected in enumerate(expected_list):
while True:
task = self.task_m.consume_task()
if task is None:
break
task.fire_callbacks()
logger.debug('#: %d, result: %s, expected: %s' % (i,
self.callback_order, expected))
assert self.callback_order == expected
self.callback_order = []
self.task_m.add(Task(0, self.callback_f, 'a'))
self.task_m.add(Task(.5, self.callback_f, 'b'))
self.task_m.add(Task(1, self.callback_f, 'c'))
time.sleep(.5)
tasks[9].cancel() # too late (already fired)
tasks[14].cancel() # should be cancelled
def test_cancel(self):
for i in xrange(5):
self.task_m.add(Task(.1, self.callback_f, i))
c_task = Task(.1, self.callback_f, 5)
self.task_m.add(c_task)
for i in xrange(6,10):
self.task_m.add(Task(.1, self.callback_f, i))
while True:
task = self.task_m.consume_task()
if task is None:
break
task.fire_callback()
logger.debug('%s' % self.callback_order)
assert self.callback_order == []
ok_(not c_task.cancelled)
c_task.cancel()
ok_(c_task.cancelled)
time.sleep(.1)
while True:
task = self.task_m.consume_task()
if task is None:
break
task.fire_callbacks()
logger.debug('%s' % self.callback_order)
assert self.callback_order == [0,1,2,3,4, 6,7,8,9]
def test_capture(self):
self.reactor.start_capture()
ts1 = time.time()
time.sleep(tc.TASK_INTERVAL/2)
# out > DATAGRAM1 (main_loop)
self.reactor.run_one_step()
ts2 = time.time()
incoming_datagram = Datagram(DATA1, tc.SERVER_ADDR)
self.reactor.s.put_datagram_received(incoming_datagram)
time.sleep(tc.TASK_INTERVAL/2)
self.reactor.run_one_step()
# in < incoming_datagram (socket)
# out > DATAGRAM3 (on_datagram_received)
captured_msgs = self.reactor.stop_and_get_capture()
eq_(len(captured_msgs), 3)
for msg in captured_msgs:
print msg
assert ts1 < captured_msgs[0][0] < ts2
eq_(captured_msgs[0][1], tc.SERVER_ADDR)
eq_(captured_msgs[0][2], True) #outgoing
eq_(captured_msgs[0][3], DATA1)
assert captured_msgs[1][0] > ts2
eq_(captured_msgs[1][1], DATAGRAM1.addr)
eq_(captured_msgs[1][2], False) #incoming
eq_(captured_msgs[1][3], DATAGRAM1.data)
assert captured_msgs[2][0] > captured_msgs[1][0]
eq_(captured_msgs[2][1], DATAGRAM3.addr)
eq_(captured_msgs[2][2], True) #outgoing
eq_(captured_msgs[2][3], DATAGRAM3.data)
def stop(self):
self._lock.acquire()
try:
self.stop_flag = True
finally:
self._lock.release()
time.sleep(self.task_interval)
def test_get_expired_value(self):
self.t.put(KEYS[0], PEERS[0])
eq_(self.t.num_keys, 1)
eq_(self.t.num_peers, 1)
time.sleep(30)
eq_(self.t.get(KEYS[0]), [])
#eq_(self.t.num_keys, 0)
eq_(self.t.num_peers, 0)
def test_network_callback(self):
self.client_r.sendto(DATA, tc.SERVER_ADDR)
time.sleep(tc.TASK_INTERVAL)
with self.lock:
first_datagram = self.datagrams_received.pop(0)
logger.debug('first_datagram: %s, %s' % (
first_datagram,
(DATA, tc.CLIENT_ADDR)))
assert first_datagram, (DATA, tc.CLIENT_ADDR)
def test_callback_list(self):
self.task_m.add(Task(tc.TASK_INTERVAL/2,
[self._callback1, self._callback2],
1, 2))
ok_(self.task_m.consume_task() is None)
eq_(self.callback_order, [])
time.sleep(tc.TASK_INTERVAL)
self.task_m.consume_task().fire_callbacks()
eq_(self.callback_order, [1,2])
def stop(self):
"""Stop the thread. It cannot be resumed afterwards????"""
#with self._lock:
self._lock.acquire()
try:
self.stop_flag = True
finally:
self._lock.release()
# wait a little for the thread to end
time.sleep(self.task_interval)
def test_call_main_loop(self):
time.sleep(tc.TASK_INTERVAL)
# main_loop is called right away
with self.lock:
# FIXME: this assert fails sometimes!!!!
eq_(self.main_loop_call_counter, 1)
time.sleep(0.1 + tc.TASK_INTERVAL)
with self.lock:
# FIXME: this crashes when recompiling
eq_(self.main_loop_call_counter, 2)
def recvfrom(self, buffer_size):
datagram_received = None
for i in xrange(9):
time.sleep(tc.TASK_INTERVAL / 10)
with self.lock:
if self.datagrams_received:
datagram_received = self.datagrams_received.pop(0)
if datagram_received:
return (datagram_received.data, datagram_received.addr)
raise socket.timeout
def test_sendto(self):
logger.critical('TESTING: IGNORE CRITICAL MESSAGE')
assert not self.main_loop_send_called
# self.r.start()
while not self.r.running:
time.sleep(tc.TASK_INTERVAL)
while not self.main_loop_send_called:
time.sleep(tc.TASK_INTERVAL)
assert self.r.s.error_raised
assert self.r.running # reactor doesn't crashed
def test_recvfrom(self):
self.r.start()
r2 = ThreadedReactor()
r2.listen_udp(tc.SERVER_ADDR[1], lambda x,y:None)
logger.critical('TESTING: IGNORE CRITICAL MESSAGE')
r2.sendto('z', tc.CLIENT_ADDR)
# self.r will call recvfrom (which raises socket.error)
time.sleep(tc.TASK_INTERVAL)
ok_(not self.callback_fired)
self.r.stop()
def test_failed_join(self):
self.lock = threading.RLock()
self.reactor = ThreadedReactor(self._main_loop,
tc.CLIENT_PORT,
self._on_datagram_received,
task_interval=tc.TASK_INTERVAL)
self.reactor.s = _SocketMock(tc.TASK_INTERVAL)
# self.reactor.start()
self.reactor.call_asap(self._very_long_callback)
time.sleep(tc.TASK_INTERVAL*2)
assert_raises(Exception, self.reactor.stop)
def test_on_datagram_received_callback(self):
# This is equivalent to sending a datagram to reactor
self.s.put_datagram_received(Datagram(DATA1, tc.SERVER_ADDR))
datagram = Datagram(DATA1, tc.SERVER_ADDR)
print "--------------", datagram, datagram.data, datagram.addr
time.sleep(tc.TASK_INTERVAL * 1)
with self.lock:
datagram = self.datagrams_received.pop(0)
print "popped>>>>>>>>>>>>>>>", datagram
eq_(datagram.data, DATA1)
eq_(datagram.addr, tc.SERVER_ADDR)
def stop(self):#, stop_callback):
"""Stop the thread. It cannot be resumed afterwards"""
self.running = False
self.join(self.task_interval*20)
if self.isAlive():
logger.info('minitwisted thread still alive. Wait a little more')
time.sleep(self.task_interval*20)
self.join(self.task_interval*20)
if self.isAlive():
raise Exception, 'Minitwisted thread is still alive!'
def test_ping_with_timeout(self):
# Client creates a query
ping_msg = clients_msg_f.outgoing_ping_query(tc.SERVER_NODE)
q = ping_msg
# Client registers query
bencoded_msg = self.querier.register_queries([q])
# Client sends bencoded_msg
time.sleep(3)
# The server never responds and the timeout is triggered
timeout_queries = self.querier.get_timeout_queries()
eq_(len(timeout_queries[1]), 1)
assert timeout_queries[1][0] is ping_msg
def test_call_main_loop(self):
eq_(self.main_loop_call_counter, 0)
self.reactor.run_one_step()
# main_loop is called right away
eq_(self.main_loop_call_counter, 1)
self.reactor.run_one_step()
# no events
eq_(self.main_loop_call_counter, 1)
time.sleep(self.main_loop_delay)
self.reactor.run_one_step()
# main_loop is called again after
eq_(self.main_loop_call_counter, 2)
def test_call_asap(self):
with self.lock:
eq_(self.callback_values, [])
self.reactor.call_asap(self._callback, 0)
time.sleep(tc.TASK_INTERVAL * 2)
with self.lock:
eq_(self.callback_values, [0])
for i in xrange(1, 5):
self.reactor.call_asap(self._callback, i)
time.sleep(tc.TASK_INTERVAL * 3)
with self.lock:
eq_(self.callback_values, range(i + 1))
def test_block_flood(self):
from floodbarrier import MAX_PACKETS_PER_PERIOD as FLOOD_LIMIT
for _ in xrange(FLOOD_LIMIT):
self.s.put_datagram_received(Datagram(DATA1, tc.SERVER_ADDR))
time.sleep(tc.TASK_INTERVAL * 5)
with self.lock:
eq_(len(self.datagrams_received), FLOOD_LIMIT)
for _ in xrange(10):
self.s.put_datagram_received(Datagram(DATA1, tc.SERVER_ADDR))
time.sleep(tc.TASK_INTERVAL * 3)
with self.lock:
eq_(len(self.datagrams_received), FLOOD_LIMIT)
logger.warning("TESTING LOGS ** IGNORE EXPECTED WARNING **")
def test_error_received(self):
# Client creates a query
msg = message.OutgoingPingQuery(tc.SERVER_NODE, tc.CLIENT_ID)
q = msg
# Client registers query
bencoded_msg = self.querier.register_queries([q])
# Client sends bencoded_msg
time.sleep(1)
# Server gets bencoded_msg and creates response
ping_r_msg_out = message.OutgoingErrorMsg(tc.CLIENT_NODE, message.GENERIC_E)
bencoded_r = ping_r_msg_out.stamp(msg.tid)
# The client receives the bencoded message
ping_r_in = message.IncomingMsg(Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is msg
def _test_recvfrom(self):
#self.r.start()
r2 = ThreadedReactor(self._main_loop, tc.CLIENT_PORT,
self._on_datagram_received,
task_interval=tc.TASK_INTERVAL)
r2.s = _SocketErrorMock()
assert not r2.running
# r2.start()
assert r2.running
logger.critical('TESTING: IGNORE CRITICAL MESSAGE')
# self.r will call recvfrom (which raises socket.error)
while not r2.s.error_raised:
time.sleep(tc.TASK_INTERVAL)
assert r2.running # the error is ignored
ok_(not self.callback_fired)
def test_start_and_stop(self):
'''
NOTE:
This is the only test using real threading
'''
self.reactor = ThreadedReactor(self._main_loop,
tc.CLIENT_PORT,
self._on_datagram_received,
task_interval=tc.TASK_INTERVAL)
ok_(not self.reactor.running)
self.reactor.start()
time.sleep(.1)
ok_(self.reactor.running)
self.reactor.stop()
ok_(not self.reactor.running)
def test_response_with_different_tid(self):
# Client creates a query
ping_msg = message.OutgoingPingQuery(tc.SERVER_NODE, tc.CLIENT_ID)
q = ping_msg
# Client registers query
bencoded_msg = self.querier.register_queries([q])
# Client sends bencoded_msg
time.sleep(1)
# Server gets bencoded_msg and creates response
ping_r_msg_out = message.OutgoingPingResponse(tc.CLIENT_NODE, tc.SERVER_ID)
bencoded_r = ping_r_msg_out.stamp("zz")
# The client receives the bencoded message
ping_r_in = message.IncomingMsg(Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is None
def test_does_not_leak_memory(self):
for i in range(CLEANUP_COUNTER - 1):
self.t.put(KEYS[0], PEERS[i])
eq_(self.t.num_keys, 1)
eq_(self.t.num_peers, CLEANUP_COUNTER - 1)
# This sleep makes all entries expired
time.sleep(30)
# But does not trigger any cleaning up
eq_(self.t.num_keys, 1)
eq_(self.t.num_peers, CLEANUP_COUNTER - 1)
# This put triggers a clean up
self.t.put(KEYS[1], PEERS[0])
# The expired entries are no more
eq_(self.t.num_keys, 1)
eq_(self.t.num_peers, 1)
def test_many_queries(self):
# Client creates a query
msgs = [clients_msg_f.outgoing_ping_query(
tc.SERVER_NODE) for i in xrange(10)]
queries = msgs
# Client registers query
bencoded_msg = self.querier.register_queries(queries)
# Client sends bencoded_msg
time.sleep(1)
# response for queries[3]
ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE)
bencoded_r = ping_r_msg_out.stamp(msgs[3].tid)
ping_r_in = clients_msg_f.incoming_msg(
Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is msgs[3]
# error for queries[2]
ping_r_msg_out = servers_msg_f.outgoing_error(tc.CLIENT_NODE,
message.GENERIC_E)
bencoded_r = ping_r_msg_out.stamp(msgs[2].tid)
ping_r_in = clients_msg_f.incoming_msg(
Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is msgs[2]
# response to wrong addr
ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE)
bencoded_r = ping_r_msg_out.stamp(msgs[5].tid)
ping_r_in = clients_msg_f.incoming_msg(
Datagram(bencoded_r, tc.SERVER2_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is None
# response with wrong tid
ping_r_msg_out = servers_msg_f.outgoing_ping_response(tc.CLIENT_NODE)
bencoded_r = ping_r_msg_out.stamp('ZZ')
ping_r_in = clients_msg_f.incoming_msg(
Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is None
# Still no time to trigger timeouts
eq_(self.querier.get_timeout_queries()[1], [])
time.sleep(1)
# Now, the timeouts can be triggered
timeout_queries = self.querier.get_timeout_queries()
expected_msgs = msgs[:2] + msgs[4:]
eq_(len(timeout_queries[1]), len(expected_msgs))
for related_query, expected_msg in zip(
timeout_queries[1], expected_msgs):
assert related_query is expected_msg
def test_ping_with_reponse(self):
# Client creates a query
ping_msg = message.OutgoingPingQuery(tc.SERVER_NODE, tc.CLIENT_ID)
q = ping_msg
# Client registers query
timeout_ts, bencoded_msgs = self.querier.register_queries([q])
# Client sends bencoded_msg
# Server gets bencoded_msg and creates response
ping_r_msg_out = message.OutgoingPingResponse(tc.CLIENT_NODE, tc.SERVER_ID)
bencoded_r = ping_r_msg_out.stamp(ping_msg.tid)
time.sleep(1)
eq_(self.querier.get_timeout_queries()[1], [])
# The client receives the bencoded message (after 1 second)
ping_r_in = message.IncomingMsg(Datagram(bencoded_r, tc.SERVER_ADDR))
related_query = self.querier.get_related_query(ping_r_in)
assert related_query is ping_msg
def test_simple(self):
for i in xrange(5):
self.task_m.add(Task(.01, self.callback_f, i))
while True:
task = self.task_m.consume_task()
if task is None:
break
task.fire_callback()
logger.debug('%s' % self.callback_order)
assert self.callback_order == []
time.sleep(.01)
while True:
task = self.task_m.consume_task()
if task is None:
break
task.fire_callbacks()
assert self.callback_order == range(5)
def test_block_flood(self):
from floodbarrier import MAX_PACKETS_PER_PERIOD as FLOOD_LIMIT
for _ in xrange(FLOOD_LIMIT):
self.client_r.sendto(DATA, tc.SERVER_ADDR)
for _ in xrange(10):
self.client_r.sendto(DATA, tc.SERVER_ADDR)
logger.warning(
"TESTING LOGS ** IGNORE EXPECTED WARNING **")
time.sleep(tc.TASK_INTERVAL)
return
######################################
with self.lock:
logger.debug('datagram processed: %d/%d' % (
len(self.datagrams_received),
FLOOD_LIMIT))
print len(self.datagrams_received)
assert len(self.datagrams_received) <= FLOOD_LIMIT
def test_listen_upd(self):
r = ThreadedReactor()
r.start()
logger.warning(''.join(
('TESTING LOGS ** IGNORE EXPECTED WARNING ** ',
'(udp_listen has not been called)')))
self.client_r.sendto(DATA, tc.SERVER_ADDR)
while 1: #waiting for data
with self.lock:
if self.datagrams_received:
break
time.sleep(tc.TASK_INTERVAL)
with self.lock:
first_datagram = self.datagrams_received.pop(0)
logger.debug('first_datagram: %s, %s' % (
first_datagram,
(DATA, tc.CLIENT_ADDR)))
assert first_datagram, (DATA, tc.CLIENT_ADDR)
r.stop()
def run(self):
"""Main loop activated by calling self.start()"""
last_task_run = time.time()
stop_flag = self.stop_flag
while not stop_flag:
timeout_raised = False
try:
data, addr = self.s.recvfrom(BUFFER_SIZE)
except (AttributeError):
logger.warning('udp_listen has not been called')
time.sleep(self.task_interval)
#TODO2: try using Event and wait
timeout_raised = True
except (socket.timeout):
timeout_raised = True
except (socket.error), e:
logger.critical(
'Got socket.error when receiving (more info follows)')
logger.exception('See critical log above')
else:
ip_is_blocked = self.floodbarrier_active and \
self.floodbarrier.ip_blocked(addr[0])
if ip_is_blocked:
logger.warning('%s blocked' % `addr`)
else:
self.datagram_received_f(data, addr)
if timeout_raised or \
time.time() - last_task_run > self.task_interval:
#with self._lock:
self._lock.acquire()
try:
while True:
task = self.tasks.consume_task()
if task is None:
break
# logger.critical('TASK COUNT 2 %d' % sys.getrefcount(task))
task.fire_callbacks()
stop_flag = self.stop_flag
finally:
self._lock.release()
def _very_long_callback(self, value):
time.sleep(tc.TASK_INTERVAL*11)
return []
def stop(self):
"""Stop the DHT."""
self.controller.stop()
time.sleep(.1) # Give time for the controller (reactor) to stop
def test_hundred_puts(self):
# test > 5 puts
eq_(self.t.num_peers, 0)
time.sleep(0)
eq_(self.t.num_peers, 0)
self.t.put(1, 1)
eq_(self.t.num_peers, 1)
time.sleep(25)
eq_(self.t.num_peers, 1)
self.t.put(2, 2)
eq_(self.t.num_peers, 2)
time.sleep(1)
eq_(self.t.num_peers, 2)
self.t.put(3, 3)
eq_(self.t.num_peers, 3)
time.sleep(1)
eq_(self.t.num_peers, 3)
self.t.put(4, 4)
eq_(self.t.num_peers, 4)
time.sleep(3)
eq_(self.t.num_peers, 4)
self.t.put(5, 5)
# cleaning... 1 out
eq_(self.t.num_peers, 4)
time.sleep(.0)
eq_(self.t.num_peers, 4)
self.t.put(6, 6)
eq_(self.t.num_peers, 5)
time.sleep(.00)
eq_(self.t.num_peers, 5)
self.t.put(7, 7)
eq_(self.t.num_peers, 6)
time.sleep(20)
eq_(self.t.num_peers, 6)
self.t.put(8, 8)
eq_(self.t.num_peers, 7)
time.sleep(.00)
eq_(self.t.num_peers, 7)
self.t.put(9, 9)
eq_(self.t.num_peers, 8)
time.sleep(10)
eq_(self.t.num_peers, 8)
self.t.put(0, 0)
# cleaning ... 2,3,4,5,6,7 out
eq_(self.t.num_peers, 3)
logging.critical('argv %r' % sys.argv)
RUN_DHT = True
my_addr = ('192.16.125.242', 2222) # (sys.argv[1], int(sys.argv[2])) #
logs_path = '.' # sys.argv[3]
print 'logs_path:', logs_path
logging_conf.setup(logs_path, logs_level)
dht = kadtracker.KadTracker(my_addr, logs_path)
else:
RUN_DHT = False
print 'usage: python server_dht.py dht_ip dht_port path'
try:
print 'Type Control-C to exit.'
i = 0
if RUN_DHT:
time.sleep(10 * 60)
for i, info_hash in enumerate(info_hashes):
# splitted_heap_str = str(hp.heap()).split()
# print i, splitted_heap_str[10]
# dht.print_routing_table_stats()
print '>>>>>>>>>>>>>>>>>>> [%d] Lookup:' % (i)
dht.get_peers(info_hash, peers_found)
time.sleep(1 * 60)
# time.sleep(1.5)
# dht.stop()
# pdb.set_trace()
i = i + 1
time.sleep(10 * 60)
dht.stop()
except (KeyboardInterrupt):
dht.stop()
def stop(self):
self.controller.stop()
time.sleep(0.1)
def _very_long_callback(self):
time.sleep(tc.TASK_INTERVAL*15)
return time.time() + 100, []