class FakeCallback():
def __init__(self):
from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
self.queue = TimedTaskQueue("FakeCallback")
def register(self,
call,
args=(),
kargs=None,
delay=0.0,
priority=0,
id_=u"",
callback=None,
callback_args=(),
callback_kargs=None,
include_id=False):
def do_task():
if kargs:
call(*args, **kargs)
else:
call(*args)
if callback:
if callback_kargs:
callback(*callback_args, **callback_kargs)
else:
callback(*callback_args)
self.queue.add_task(do_task, t=delay)
def shutdown(self, immediately=False):
self.queue.shutdown(immediately)
class TestBuddyCastDataHandler(unittest.TestCase):
def setUp(self):
# prepare database
launchmany = FakeLauchMany()
self.overlay_bridge = TimedTaskQueue()
#self.overlay_bridge = FakeOverlayBridge()
self.data_handler = DataHandler(launchmany, self.overlay_bridge, max_num_peers=2500)
def tearDown(self):
self.overlay_bridge.add_task('quit')
def test_postInit(self):
#self.data_handler.postInit()
self.data_handler.postInit(1,50,0, 50)
class TestBuddyCastDataHandler(unittest.TestCase):
def setUp(self):
# prepare database
launchmany = FakeLauchMany()
self.overlay_bridge = TimedTaskQueue()
#self.overlay_bridge = FakeOverlayBridge()
self.data_handler = DataHandler(launchmany,
self.overlay_bridge,
max_num_peers=2500)
def tearDown(self):
self.overlay_bridge.add_task('quit')
def test_postInit(self):
#self.data_handler.postInit()
self.data_handler.postInit(1, 50, 0, 50)
class FakeCallback():
def __init__(self):
from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
self.queue = TimedTaskQueue("FakeCallback")
def register(self, call, args=(), kargs=None, delay=0.0, priority=0, id_=u"", callback=None, callback_args=(), callback_kargs=None, include_id=False):
def do_task():
if kargs:
call(*args, **kargs)
else:
call(*args)
if callback:
if callback_kargs:
callback(*callback_args, **callback_kargs)
else:
callback(*callback_args)
self.queue.add_task(do_task, t=delay)
def shutdown(self, immediately=False):
self.queue.shutdown(immediately)
class TestGUITaskQueue(unittest.TestCase):
def setUp(self):
self.ntasks = 0
self.completed = []
self.guiserver = TimedTaskQueue()
def tearDown(self):
sleep(2)
self.completed.sort()
if self.completed != range(self.ntasks):
print "test failed",self.completed
self.assert_(False)
def test_simple(self):
self.ntasks = 1
self.guiserver.add_task(lambda:self.task(0),0)
def test_more(self):
self.ntasks = 10
for i in range(self.ntasks):
# lambda functions are evil, this is not the same as lambda:task(i)
self.guiserver.add_task(self.define_task(i),0)
def test_delay(self):
self.ntasks = 1
self.guiserver.add_task(lambda:self.task(0),3)
print "test: sleeping 5 secs so tasks gets executed"
sleep(5)
def test_delay2(self):
self.ntasks = 2
self.guiserver.add_task(lambda:self.task(1),3)
self.guiserver.add_task(lambda:self.task(0),1)
print "test: sleeping 5 secs so tasks gets executed"
sleep(5)
def define_task(self,num):
return lambda:self.task(num)
def task(self,num):
print "Running task",num
self.completed.append(num)
class TestGUITaskQueue(unittest.TestCase):
def setUp(self):
self.ntasks = 0
self.completed = []
self.guiserver = TimedTaskQueue()
def tearDown(self):
sleep(2)
self.completed.sort()
if self.completed != range(self.ntasks):
print "test failed", self.completed
self.assert_(False)
def test_simple(self):
self.ntasks = 1
self.guiserver.add_task(lambda: self.task(0), 0)
def test_more(self):
self.ntasks = 10
for i in range(self.ntasks):
# lambda functions are evil, this is not the same as lambda:task(i)
self.guiserver.add_task(self.define_task(i), 0)
def test_delay(self):
self.ntasks = 1
self.guiserver.add_task(lambda: self.task(0), 3)
print "test: sleeping 5 secs so tasks gets executed"
sleep(5)
def test_delay2(self):
self.ntasks = 2
self.guiserver.add_task(lambda: self.task(1), 3)
self.guiserver.add_task(lambda: self.task(0), 1)
print "test: sleeping 5 secs so tasks gets executed"
sleep(5)
def define_task(self, num):
return lambda: self.task(num)
def task(self, num):
print "Running task", num
self.completed.append(num)
class TestTimedTaskQueue(unittest.TestCase):
def setUp(self):
self.queue = TimedTaskQueue()
def tearDown(self):
self.queue.shutdown()
del self.queue
def test_addTask(self):
self.count = 0
self.queue.add_task(self.task3a, 3)
self.queue.add_task(self.task0, 0)
self.queue.add_task(self.task3b, 3)
self.queue.add_task(self.task2, 1)
sleep(6)
assert self.count == 11
def task0(self):
self.count += 1
assert self.count == 1
def task2(self):
self.count += 2
assert self.count == 3
def task3a(self):
self.count += 4
assert self.count == 7 or self.count == 11
def task3b(self):
self.count += 4
assert self.count == 7 or self.count == 11
def test_addTask0FIFO(self):
self.count = 0
self.queue.add_task(self.task0a, 0)
self.queue.add_task(self.task0b, 0)
self.queue.add_task(self.task0c, 0)
self.queue.add_task(self.task0d, 0)
sleep(6)
assert self.count == 4
def task0a(self):
assert self.count == 0
self.count = 1
def task0b(self):
assert self.count == 1
self.count = 2
def task0c(self):
assert self.count == 2
self.count = 3
def task0d(self):
assert self.count == 3
self.count = 4
class OverlayThreadingBridge:
__single = None
lock = threading.Lock()
def __init__(self):
if OverlayThreadingBridge.__single:
raise RuntimeError, "OverlayThreadingBridge is Singleton"
OverlayThreadingBridge.__single = self
self.secover = None
self.olapps = None
self.olappsmsghandler = None
self.olappsconnhandler = None
# Current impl of wrapper: single thread
self.tqueue = TimedTaskQueue(nameprefix="Overlay")
self.gcqueue = TimedTaskQueue(nameprefix="GameCast")
def getInstance(*args, **kw):
# Singleton pattern with double-checking
if OverlayThreadingBridge.__single is None:
OverlayThreadingBridge.lock.acquire()
try:
if OverlayThreadingBridge.__single is None:
OverlayThreadingBridge(*args, **kw)
finally:
OverlayThreadingBridge.lock.release()
return OverlayThreadingBridge.__single
getInstance = staticmethod(getInstance)
def resetSingleton(self):
""" For testing purposes """
OverlayThreadingBridge.__single = None
def register_bridge(self,secover,olapps):
""" Called by MainThread """
self.secover = secover
self.olapps = olapps
secover.register_recv_callback(self.handleMessage)
secover.register_conns_callback(self.handleConnection)
#
# SecOverlay interface
#
def register(self,launchmanycore,max_len):
""" Called by MainThread """
self.secover.register(launchmanycore,max_len)
# FOR TESTING ONLY
self.iplport2oc = self.secover.iplport2oc
def get_handler(self):
return self.secover
def start_listening(self):
""" Called by MainThread """
self.secover.start_listening()
def register_recv_callback(self,callback):
""" Called by MainThread """
self.olappsmsghandler = callback
def register_conns_callback(self,callback):
""" Called by MainThread """
self.olappsconnhandler = callback
def handleConnection(self,exc,permid,selversion,locally_initiated,hisdns):
""" Called by NetworkThread """
# called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks()
if DEBUG:
print >>sys.stderr,"olbridge: handleConnection",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName()
def olbridge_handle_conn_func():
# Called by OverlayThread
if DEBUG:
print >>sys.stderr,"olbridge: handle_conn_func",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName()
try:
if hisdns:
self.secover.add_peer_to_db(permid,hisdns,selversion)
if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection
self.olappsconnhandler(exc,permid,selversion,locally_initiated)
except:
print_exc()
if isinstance(exc,CloseException):
self.secover.update_peer_status(permid,exc.was_auth_done())
self.tqueue.add_task(olbridge_handle_conn_func,0)
def handleMessage(self,permid,selversion,message):
""" Called by NetworkThread """
#ProxyService_
#
# DEBUG
#print "### olbridge: handleMessage", show_permid_short(permid), selversion, getMessageName(message[0]), currentThread().getName()
#
#_ProxyService
if DEBUG:
print >>sys.stderr,"olbridge: handleMessage",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
def olbridge_handle_msg_func():
# Called by OverlayThread
if DEBUG:
print >>sys.stderr,"olbridge: handle_msg_func",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
try:
if self.olappsmsghandler is None:
ret = True
else:
ret = self.olappsmsghandler(permid,selversion,message)
except:
print_exc()
ret = False
if ret == False:
if DEBUG:
print >>sys.stderr,"olbridge: olbridge_handle_msg_func closing!",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
self.close(permid)
if message[0] in GameCastMessages:
self.gcqueue.add_task(olbridge_handle_msg_func,0)
else:
self.tqueue.add_task(olbridge_handle_msg_func,0)
return True
def connect_dns(self,dns,callback):
""" Called by OverlayThread/NetworkThread """
if DEBUG:
print >>sys.stderr,"olbridge: connect_dns",dns
def olbridge_connect_dns_callback(cexc,cdns,cpermid,cselver):
# Called by network thread
if DEBUG:
print >>sys.stderr,"olbridge: connect_dns_callback",cexc,cdns,show_permid_short(cpermid),cselver
olbridge_connect_dns_callback_lambda = lambda:callback(cexc,cdns,cpermid,cselver)
self.add_task(olbridge_connect_dns_callback_lambda,0)
self.secover.connect_dns(dns,olbridge_connect_dns_callback)
def connect(self,permid,callback,gamecast = False):
""" Called by OverlayThread """
if DEBUG:
print >>sys.stderr,"olbridge: connect",show_permid_short(permid), currentThread().getName()
def olbridge_connect_callback(cexc,cdns,cpermid,cselver):
# Called by network thread
if DEBUG:
print >>sys.stderr,"olbridge: connect_callback",cexc,cdns,show_permid_short(cpermid),cselver, callback, currentThread().getName()
olbridge_connect_callback_lambda = lambda:callback(cexc,cdns,cpermid,cselver)
# Jie: postpone to call this callback to schedule it after the peer has been added to buddycast connection list
# Arno, 2008-09-15: No-no-no
if gamecast:
self.gcqueue.add_task(olbridge_connect_callback_lambda,0)
else:
self.add_task(olbridge_connect_callback_lambda,0)
self.secover.connect(permid,olbridge_connect_callback)
def send(self,permid,msg,callback,gamecast = False):
""" Called by OverlayThread """
if DEBUG:
print >>sys.stderr,"olbridge: send",show_permid_short(permid),len(msg)
def olbridge_send_callback(cexc,cpermid):
# Called by network thread
if DEBUG:
print >>sys.stderr,"olbridge: send_callback",cexc,show_permid_short(cpermid)
olbridge_send_callback_lambda = lambda:callback(cexc,cpermid)
if gamecast:
self.gcqueue.add_task(olbridge_send_callback_lambda,0)
else:
self.add_task(olbridge_send_callback_lambda,0)
self.secover.send(permid,msg,olbridge_send_callback)
def close(self,permid):
""" Called by OverlayThread """
self.secover.close(permid)
def add_task(self,task,t=0,ident=None, gamecast = False):
""" Called by OverlayThread """
if gamecast:
self.gcqueue.add_task(task,t,ident)
else:
self.tqueue.add_task(task,t,ident)
class TestBuddyCast(unittest.TestCase):
def setUp(self):
# prepare database
launchmany = FakeLauchMany()
self.overlay_bridge = TimedTaskQueue()
#self.overlay_bridge = FakeOverlayBridge()
superpeer = False # enable it to test superpeer
self.bc = BuddyCastFactory.getInstance(superpeer=superpeer)
self.bc.register(self.overlay_bridge, launchmany, None, None, None,
True)
def tearDown(self):
self.overlay_bridge.add_task('quit')
print "Before join"
def remove_t_index(self):
indices = [
'Torrent_length_idx',
'Torrent_creation_date_idx',
'Torrent_relevance_idx',
'Torrent_num_seeders_idx',
'Torrent_num_leechers_idx',
#'Torrent_name_idx',
]
for index in indices:
sql = 'drop index ' + index
self.data_handler.torrent_db._db.execute_write(sql)
def remove_p_index(self):
indices = [
'Peer_name_idx', 'Peer_ip_idx', 'Peer_similarity_idx',
'Peer_last_seen_idx', 'Peer_last_connected_idx',
'Peer_num_peers_idx', 'Peer_num_torrents_idx'
]
for index in indices:
sql = 'drop index ' + index
self.data_handler.peer_db._db.execute_write(sql)
def local_test(self):
self.remove_t_index()
self.remove_p_index()
from Tribler.Test.log_parser import get_buddycast_data
#start_time = time()
#print >> sys.stderr, "buddycast: ******************* start local test"
costs = []
self.data_handler.postInit(updatesim=False)
for permid, selversion, msg in get_buddycast_data(
os.path.join(FILES_DIR, 'superpeer120070902sp7001.log')):
message = bencode(msg)
#print 'got msg:', permid, selversion, message
try:
s = time()
self.bc.gotBuddyCastMessage(message, permid, selversion)
cost = time() - s
costs.append(cost)
except:
print_exc()
break
print 'got msg: %d %.2f %.2f %.2f %.2f' % (len(costs), cost,
min(costs), sum(costs) /
len(costs), max(costs))
# with all indices, min/avg/max: 0.00 1.78 4.57 seconds
# without index, min/avg/max: 0.00 1.38 3.43 seconds (58)
print "Done"
def test_start(self):
try:
self.bc.olthread_register(start=False)
self.data_handler = self.bc.data_handler
self.local_test()
print "Sleeping for 10 secs"
sleep(10)
print "Done2"
except:
print_exc()
self.assert_(False)
class OverlayThreadingBridge:
__single = None
lock = threading.Lock()
def __init__(self):
if OverlayThreadingBridge.__single:
raise RuntimeError, "OverlayThreadingBridge is Singleton"
OverlayThreadingBridge.__single = self
self.secover = None
self.olapps = None
self.olappsmsghandler = None
self.olappsconnhandler = None
# Current impl of wrapper: single thread
self.tqueue = TimedTaskQueue(nameprefix="Overlay")
def getInstance(*args, **kw):
# Singleton pattern with double-checking
if OverlayThreadingBridge.__single is None:
OverlayThreadingBridge.lock.acquire()
try:
if OverlayThreadingBridge.__single is None:
OverlayThreadingBridge(*args, **kw)
finally:
OverlayThreadingBridge.lock.release()
return OverlayThreadingBridge.__single
getInstance = staticmethod(getInstance)
def resetSingleton(self):
""" For testing purposes """
OverlayThreadingBridge.__single = None
def register_bridge(self, secover, olapps):
""" Called by MainThread """
self.secover = secover
self.olapps = olapps
secover.register_recv_callback(self.handleMessage)
secover.register_conns_callback(self.handleConnection)
#
# SecOverlay interface
#
def register(self, launchmanycore, max_len):
""" Called by MainThread """
self.secover.register(launchmanycore, max_len)
# FOR TESTING ONLY
self.iplport2oc = self.secover.iplport2oc
def get_handler(self):
return self.secover
def start_listening(self):
""" Called by MainThread """
self.secover.start_listening()
def register_recv_callback(self, callback):
""" Called by MainThread """
self.olappsmsghandler = callback
def register_conns_callback(self, callback):
""" Called by MainThread """
self.olappsconnhandler = callback
def handleConnection(self, exc, permid, selversion, locally_initiated,
hisdns):
""" Called by NetworkThread """
# called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks()
if DEBUG:
print >> sys.stderr, "olbridge: handleConnection", exc, show_permid_short(
permid), selversion, locally_initiated, hisdns, currentThread(
).getName()
def olbridge_handle_conn_func():
# Called by OverlayThread
if DEBUG:
print >> sys.stderr, "olbridge: handle_conn_func", exc, show_permid_short(
permid
), selversion, locally_initiated, hisdns, currentThread(
).getName()
try:
if hisdns:
self.secover.add_peer_to_db(permid, hisdns, selversion)
if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection
self.olappsconnhandler(exc, permid, selversion,
locally_initiated)
except:
print_exc()
if isinstance(exc, CloseException):
self.secover.update_peer_status(permid, exc.was_auth_done())
self.tqueue.add_task(olbridge_handle_conn_func, 0)
def handleMessage(self, permid, selversion, message):
""" Called by NetworkThread """
#ProxyService_
#
# DEBUG
#print "### olbridge: handleMessage", show_permid_short(permid), selversion, getMessageName(message[0]), currentThread().getName()
#
#_ProxyService
if DEBUG:
print >> sys.stderr, "olbridge: handleMessage", show_permid_short(
permid), selversion, getMessageName(
message[0]), currentThread().getName()
def olbridge_handle_msg_func():
# Called by OverlayThread
if DEBUG:
print >> sys.stderr, "olbridge: handle_msg_func", show_permid_short(
permid), selversion, getMessageName(
message[0]), currentThread().getName()
try:
if self.olappsmsghandler is None:
ret = True
else:
ret = self.olappsmsghandler(permid, selversion, message)
except:
print_exc()
ret = False
if ret == False:
if DEBUG:
print >> sys.stderr, "olbridge: olbridge_handle_msg_func closing!", show_permid_short(
permid), selversion, getMessageName(
message[0]), currentThread().getName()
self.close(permid)
self.tqueue.add_task(olbridge_handle_msg_func, 0)
return True
def connect_dns(self, dns, callback):
""" Called by OverlayThread/NetworkThread """
if DEBUG:
print >> sys.stderr, "olbridge: connect_dns", dns
def olbridge_connect_dns_callback(cexc, cdns, cpermid, cselver):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: connect_dns_callback", cexc, cdns, show_permid_short(
cpermid), cselver
olbridge_connect_dns_callback_lambda = lambda: callback(
cexc, cdns, cpermid, cselver)
self.add_task(olbridge_connect_dns_callback_lambda, 0)
self.secover.connect_dns(dns, olbridge_connect_dns_callback)
def connect(self, permid, callback):
""" Called by OverlayThread """
if DEBUG:
print >> sys.stderr, "olbridge: connect", show_permid_short(
permid), currentThread().getName()
def olbridge_connect_callback(cexc, cdns, cpermid, cselver):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: connect_callback", cexc, cdns, show_permid_short(
cpermid), cselver, callback, currentThread().getName()
olbridge_connect_callback_lambda = lambda: callback(
cexc, cdns, cpermid, cselver)
# Jie: postpone to call this callback to schedule it after the peer has been added to buddycast connection list
# Arno, 2008-09-15: No-no-no
self.add_task(olbridge_connect_callback_lambda, 0)
self.secover.connect(permid, olbridge_connect_callback)
def send(self, permid, msg, callback):
""" Called by OverlayThread """
if DEBUG:
print >> sys.stderr, "olbridge: send", show_permid_short(
permid), len(msg)
def olbridge_send_callback(cexc, cpermid):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: send_callback", cexc, show_permid_short(
cpermid)
olbridge_send_callback_lambda = lambda: callback(cexc, cpermid)
self.add_task(olbridge_send_callback_lambda, 0)
self.secover.send(permid, msg, olbridge_send_callback)
def close(self, permid):
""" Called by OverlayThread """
self.secover.close(permid)
def add_task(self, task, t=0, ident=None):
""" Called by OverlayThread """
self.tqueue.add_task(task, t, ident)
class TestBuddyCast(unittest.TestCase):
def setUp(self):
# prepare database
launchmany = FakeLauchMany()
self.overlay_bridge = TimedTaskQueue()
#self.overlay_bridge = FakeOverlayBridge()
superpeer=False # enable it to test superpeer
self.bc = BuddyCastFactory.getInstance(superpeer=superpeer)
self.bc.register(self.overlay_bridge, launchmany, None,
None, None, True)
def tearDown(self):
self.overlay_bridge.add_task('quit')
print "Before join"
def remove_t_index(self):
indices = [
'Torrent_length_idx',
'Torrent_creation_date_idx',
'Torrent_relevance_idx',
'Torrent_num_seeders_idx',
'Torrent_num_leechers_idx',
#'Torrent_name_idx',
]
for index in indices:
sql = 'drop index ' + index
self.data_handler.torrent_db._db.execute_write(sql)
def remove_p_index(self):
indices = [
'Peer_name_idx',
'Peer_ip_idx',
'Peer_similarity_idx',
'Peer_last_seen_idx',
'Peer_last_connected_idx',
'Peer_num_peers_idx',
'Peer_num_torrents_idx'
]
for index in indices:
sql = 'drop index ' + index
self.data_handler.peer_db._db.execute_write(sql)
def local_test(self):
self.remove_t_index()
self.remove_p_index()
from Tribler.Test.log_parser import get_buddycast_data
#start_time = time()
#print >> sys.stderr, "buddycast: ******************* start local test"
costs = []
self.data_handler.postInit(updatesim=False)
for permid, selversion, msg in get_buddycast_data(os.path.join(FILES_DIR,'superpeer120070902sp7001.log')):
message = bencode(msg)
#print 'got msg:', permid, selversion, message
try:
s = time()
self.bc.gotBuddyCastMessage(message, permid, selversion)
cost = time()-s
costs.append(cost)
except:
print_exc()
break
print 'got msg: %d %.2f %.2f %.2f %.2f' %(len(costs), cost, min(costs), sum(costs)/len(costs), max(costs))
# with all indices, min/avg/max: 0.00 1.78 4.57 seconds
# without index, min/avg/max: 0.00 1.38 3.43 seconds (58)
print "Done"
def test_start(self):
try:
self.bc.olthread_register(start=False)
self.data_handler = self.bc.data_handler
self.local_test()
print "Sleeping for 10 secs"
sleep(10)
print "Done2"
except:
print_exc()
self.assert_(False)
