def setUp(self): print >> sys.stderr, "test: TestOverlayThreadingBridge.setUp()" secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() overbridge1 = OverlayThreadingBridge() overbridge1.register_bridge(secover1, None) overbridge2 = OverlayThreadingBridge() overbridge2.register_bridge(secover2, None) self.peer1 = Peer(self, 1234, overbridge1) self.peer2 = Peer(self, 5678, overbridge2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >> sys.stderr, "test: setUp: peer1 permid is", show_permid_short( self.peer1.my_permid) print >> sys.stderr, "test: setUp: peer2 permid is", show_permid_short( self.peer2.my_permid) time.sleep(2) # let server threads start
def __init__(self, infohash, num_pieces, btdownloader, proxydownloader, encoder): # Number of pieces in the torrent self.num_pieces = num_pieces # Torrent infohash self.infohash = infohash # readonly so no locking on this # List of proxies that have been asked to relay but have not confirmed yet # deque is thread-safe self.asked_proxies = deque() # List of confirmed proxies # deque is thread-safe self.confirmed_proxies = deque() # Dictionary for keeping evidence of proxies and the pieces requested to them # Key: permid of a proxy # Value: list of pieces requested to that proxy self.requested_pieces = {} # Tribler overlay warm self.overlay_bridge = OverlayThreadingBridge.getInstance() # BT1Download object self.btdownloader = btdownloader self.proxydownloader = proxydownloader # Encoder object self.encoder = encoder
def __init__(self, torrent_hash, num_pieces, coordinator_permid, coordinator = None): self.overlay_bridge = OverlayThreadingBridge.getInstance() self.torrent_hash = torrent_hash if coordinator_permid is not None and coordinator_permid == '': self.coordinator_permid = None else: self.coordinator_permid = coordinator_permid self.coordinator_ip = None # see is_coordinator() self.coordinator_port = -1 if self.coordinator_permid is not None: peerdb = PeerDBHandler.getInstance() peer = peerdb.getPeer(coordinator_permid) if peer is not None: self.coordinator_ip = peer['ip'] self.coordinator_port = peer['port'] self.reserved_pieces = [False] * num_pieces self.ignored_pieces = [False] * num_pieces self.coordinator = coordinator self.counter = 0 self.completed = False self.distr_reserved_pieces = [False] * num_pieces self.marker = [True] * num_pieces self.round = 0 self.encoder = None self.continuations = [] self.outstanding = None self.last_req_time = 0
def __init__(self, torrent_hash, num_pieces, coordinator_permid, coordinator=None): self.overlay_bridge = OverlayThreadingBridge.getInstance() self.torrent_hash = torrent_hash if coordinator_permid is not None and coordinator_permid == '': self.coordinator_permid = None else: self.coordinator_permid = coordinator_permid self.coordinator_ip = None # see is_coordinator() self.coordinator_port = -1 if self.coordinator_permid is not None: peerdb = PeerDBHandler.getInstance() peer = peerdb.getPeer(coordinator_permid) if peer is not None: self.coordinator_ip = peer['ip'] self.coordinator_port = peer['port'] self.reserved_pieces = [False] * num_pieces self.ignored_pieces = [False] * num_pieces self.coordinator = coordinator self.counter = 0 self.completed = False self.distr_reserved_pieces = [False] * num_pieces self.marker = [True] * num_pieces self.round = 0 self.encoder = None self.continuations = [] self.outstanding = None self.last_req_time = 0
def set_stop_collecting_threshold(self, value): self.sesslock.acquire() try: SessionConfigInterface.set_stop_collecting_threshold(self, value) olbridge = OverlayThreadingBridge.getInstance() task = lambda: self.olthread_set_stop_collecting_threshold(value) olbridge.add_task(task, 0) finally: self.sesslock.release()
def __init__(self): self.metadata_queue = {} self.metadata_queue_lock = Lock() self.overlay_bridge = OverlayThreadingBridge.getInstance() # active_reauests is a dictionary storing al permids asking for relay # until the download object is created. The dictionary keys are infohashes, # and the values are lists of permids self.active_requests = {} self.session = None
def set_stop_collecting_threshold(self,value): self.sesslock.acquire() try: SessionConfigInterface.set_stop_collecting_threshold(self,value) olbridge = OverlayThreadingBridge.getInstance() task = lambda:self.olthread_set_stop_collecting_threshold(value) olbridge.add_task(task,0) finally: self.sesslock.release()
def set_start_recommender(self,value): self.sesslock.acquire() try: SessionConfigInterface.set_start_recommender(self,value) olbridge = OverlayThreadingBridge.getInstance() task = lambda:self.olthread_set_start_recommender(value) olbridge.add_task(task,0) finally: self.sesslock.release()
def __init__(self, infohash, num_pieces): self.reserved_pieces = [False] * num_pieces self.infohash = infohash # readonly so no locking on this self.lock = Lock() self.asked_helpers = [] # protected by lock # optimization self.reserved = [] self.overlay_bridge = OverlayThreadingBridge.getInstance()
def set_start_recommender(self, value): self.sesslock.acquire() try: SessionConfigInterface.set_start_recommender(self, value) olbridge = OverlayThreadingBridge.getInstance() task = lambda: self.olthread_set_start_recommender(value) olbridge.add_task(task, 0) finally: self.sesslock.release()
def set_torrent_collecting_rate(self,value): self.sesslock.acquire() try: from Tribler.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge SessionConfigInterface.set_torrent_collecting_rate(self,value) olbridge = OverlayThreadingBridge.getInstance() task = lambda:self.olthread_set_torrent_collecting_rate(value) olbridge.add_task(task,0) finally: self.sesslock.release()
def set_torrent_collecting_rate(self, value): self.sesslock.acquire() try: from Tribler.Core.Overlay.OverlayThreadingBridge import OverlayThreadingBridge SessionConfigInterface.set_torrent_collecting_rate(self, value) olbridge = OverlayThreadingBridge.getInstance() task = lambda: self.olthread_set_torrent_collecting_rate(value) olbridge.add_task(task, 0) finally: self.sesslock.release()
def __init__(self, game): threading.Thread.__init__(self) self.overlay_bridge = OverlayThreadingBridge.getInstance() global session session.add_observer(self.gameCallback, NTFY_GAMECAST, [NTFY_UPDATE], 'Game') self.mypermid = session.get_permid() self.colour = game['players'][self.mypermid] self.chess = ChessBoard() self.game = game self.done = False if DEBUG: print >> sys.stderr, 'chessbot: chessbot started for game (%d,%d)' % (self.game['owner_id'], self.game['game_id'])
def setUp(self): print >>sys.stderr,"test: TestOverlayThreadingBridge.setUp()" secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() overbridge1 = OverlayThreadingBridge() overbridge1.register_bridge(secover1,None) overbridge2 = OverlayThreadingBridge() overbridge2.register_bridge(secover2,None) self.peer1 = Peer(self,1234,overbridge1) self.peer2 = Peer(self,5678,overbridge2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid) print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid) time.sleep(2) # let server threads start
def __init__(self, session): if TaskDispatcher.__single: raise RuntimeError, "TaskDispatcher is singleton" TaskDispatcher.__single = self threading.Thread.__init__(self) self.setDaemon(True) self.overlay_bridge = OverlayThreadingBridge.getInstance() self.thegame = ['e2e4', 'd7d5', 'b1c3', 'Ng8f6', 'g1f3', 'd5xe4', 'c3e4', 'Nf6xe4', 'd2d4', 'Nb8c6', \ 'c1f4', 'e7e6', 'f1b5', 'Bf8b4+', 'c2c3', 'Bb4d6', 'e1g1', 'Bd6xf4', 'd1d3', 'f7f5', \ 'b5c6', 'b7xc6', 'f3e5', 'Bf4xe5', 'd4e5', 'Qd8xd3', 'c3c4', 'Qd3d4', 'c4c5', 'Bc8a6', \ 'g2g4', 'Ba6xf1', 'g1h1', 'Qd4xf2', 'h2h4', 'Qf2g2#'] # Maximum number of games that can be played simultaneously. self.max_games = 1 # Keep all games that are being played and all received invites in memory. self.open_games = [] self.open_invites = [] # Keep a number of closed games in the cache. These games will be used when testing the discuss command. self.closed_games = [] # Register db callbacks self.session = session self.session.add_observer(self.gameCallback, NTFY_GAMECAST, [NTFY_INSERT, NTFY_UPDATE, NTFY_DELETE], 'Game') self.session.add_observer(self.inviteCallback, NTFY_GAMECAST, [NTFY_INSERT, NTFY_UPDATE, NTFY_DELETE], 'GameInvite') self.done = False self.mypermid = self.session.get_permid() self.myip = self.session.get_external_ip() self.myport = self.session.get_listen_port() self.gamecast = GameCast.getInstance() self.gamecast_gossip = GameCastGossip.getInstance() self.gamecast_database = self.session.open_dbhandler(NTFY_GAMECAST) self.peers_database = self.session.open_dbhandler(NTFY_PEERS) # For stats self.logged_peers = [] self.logged_games = [] self.start_time = time() self.gc_logger = logging.getLogger('gamecast') self.gcg_logger = logging.getLogger('gamecastgossip') self.logging = threading.Thread(target=self.periodicLogging) self.logging.daemon = True self.logging_interval = 30 self.logging.start()
def __init__(self, infohash, num_pieces, btdownloader, proxydownloader, encoder): self.infohash = infohash self.num_pieces = num_pieces # The doe_nodes is a list that stores the permids if the doe nodes self.doe_nodes = deque() # Strores the pieces requested by each permid self.requested_pieces = {} # Stores the pieces sent to each permid self.delivered_pieces = {} # Stores the pieces dropped to each permid self.dropped_pieces = {} # list of pieces requested by the doe and not passed to the bittorrent engine yet # deque is thread-safe self.current_requests = deque() # list of pieces passed to the bittorrent engine for retrieving # deque is thread-safe self.currently_downloading_pieces = deque() self.counter = 0 self.completed = False self.marker = [True] * num_pieces self.round = 0 self.btdownloader = btdownloader self.proxydownloader = proxydownloader self.encoder = encoder self.overlay_bridge = OverlayThreadingBridge.getInstance() # self.continuations = [] # self.outstanding = None self.last_req_time = 0
def __init__(self, session): if self.__singleton: raise RuntimeError, "Crawler is Singleton" self._overlay_bridge = OverlayThreadingBridge.getInstance() self._session = session self._crawler_db = CrawlerDBHandler.getInstance() # _message_handlers contains message-id:(request-callback, reply-callback, last-request-timestamp) # the handlers are called when either a CRAWL_REQUEST or CRAWL_REPLY message is received self._message_handlers = {} # _crawl_initiators is a list with (initiator-callback, # frequency, accept_frequency) tuples the initiators are called # when a new connection is received self._crawl_initiators = [] # _initiator_dealines contains [deadline, frequency, # accept_frequency, initiator-callback, permid, selversion, # failure-counter] deadlines register information on when to # call the crawl initiators again for a specific permid self._initiator_deadlines = [] # _dialback_deadlines contains message_id:(deadline, permid) pairs # client peers should connect back to -a- crawler indicated by # permid after deadline expired self._dialback_deadlines = {} # _channels contains permid:buffer-dict pairs. Where # buffer_dict contains channel-id:(timestamp, buffer, # channel_data) pairs. Where buffer is the payload from # multipart messages that are received so far. Channels are # used to match outstanding replies to given requests self._channels = {} # start checking for expired deadlines self._check_deadlines(True) # start checking for ancient channels self._check_channels()
if __name__ == "__main__": # profile_on() global orig_stdin, orig_stdout orig_stdin = sys.stdin#Unbuffered(sys.stdin) orig_stdout = sys.stdout#Unbuffered(sys.stdout) sys.stdout = sys.stderr sys.stdin = None config, fileargs = parseargs(sys.argv, argsdef, presets = {}) config['statedir'] = os.path.join(os.getcwd(), os.path.pardir, '.'+config['name']) print >> sys.stderr, "Config is", config overlay_bridge = OverlayThreadingBridge.getInstance() overlay_bridge.gcqueue = overlay_bridge.tqueue overlay_bridge.add_task(startsession, 0) params = pickle.load(orig_stdin) print 'TaskDispatcher params', params global session td = TaskDispatcher.getInstance(session) if params: td.max_games = params[0] td.start() print 'TaskDispatcher has started' while True: try: text = orig_stdin.readline()
def __init__(self, make_upload, downloader, choker, numpieces, piece_size, totalup, config, ratelimiter, merkle_torrent, sched = None, coordinator = None, helper = None, mylistenport = None, use_g2g = False, infohash=None, tracker=None): self.downloader = downloader self.make_upload = make_upload self.choker = choker self.numpieces = numpieces self.piece_size = piece_size self.config = config self.ratelimiter = ratelimiter self.rate_capped = False self.sched = sched self.totalup = totalup self.rate_capped = False self.connections = {} self.external_connection_made = 0 self.merkle_torrent = merkle_torrent self.use_g2g = use_g2g # 2fastbt_ self.coordinator = coordinator self.helper = helper self.round = 0 self.mylistenport = mylistenport self.infohash = infohash self.tracker = tracker try: (scheme, netloc, path, pars, query, fragment) = urlparse.urlparse(self.tracker) host = netloc.split(':')[0] self.tracker_ip = socket.getaddrinfo(host,None)[0][4][0] except: print_exc() self.tracker_ip = None print >>sys.stderr,"Connecter: live: source/tracker is",self.tracker_ip self.overlay_enabled = 0 if self.config['overlay']: self.overlay_enabled = True if DEBUG: if self.overlay_enabled: print >>sys.stderr,"connecter: Enabling overlay" else: print >>sys.stderr,"connecter: Disabling overlay" if DEBUG: if self.overlay_enabled: print >>sys.stderr,"connecter: Enabling overlay" else: print >>sys.stderr,"connecter: Disabling overlay" self.ut_pex_enabled = 0 if 'ut_pex_max_addrs_from_peer' in self.config: self.ut_pex_max_addrs_from_peer = self.config['ut_pex_max_addrs_from_peer'] self.ut_pex_enabled = self.ut_pex_max_addrs_from_peer > 0 self.ut_pex_previous_conns = [] # last value of 'added' field for all peers if DEBUG_UT_PEX: if self.ut_pex_enabled: print >>sys.stderr,"connecter: Enabling uTorrent PEX",self.ut_pex_max_addrs_from_peer else: print >>sys.stderr,"connecter: Disabling uTorrent PEX" # The set of messages we support. Note that the msg ID is an int not a byte in # this dict. self.EXTEND_HANDSHAKE_M_DICT = {} if self.overlay_enabled: # Say in the EXTEND handshake we support the overlay-swarm ext. d = {EXTEND_MSG_OVERLAYSWARM:ord(CHALLENGE)} self.EXTEND_HANDSHAKE_M_DICT.update(d) if self.ut_pex_enabled: # Say in the EXTEND handshake we support uTorrent's peer exchange ext. d = {EXTEND_MSG_UTORRENT_PEX:ord(EXTEND_MSG_UTORRENT_PEX_ID)} self.EXTEND_HANDSHAKE_M_DICT.update(d) self.sched(self.ut_pex_callback,6) if self.use_g2g: # Say in the EXTEND handshake we want to do G2G. d = {EXTEND_MSG_G2G_V2:ord(G2G_PIECE_XFER)} self.EXTEND_HANDSHAKE_M_DICT.update(d) self.sched(self.g2g_callback,G2G_CALLBACK_INTERVAL) # LIVEHACK livekey = EXTEND_MSG_LIVE_PREFIX+str(CURRENT_LIVE_VERSION) d = {livekey:ord(LIVE_FAKE_MESSAGE_ID)} self.EXTEND_HANDSHAKE_M_DICT.update(d) print >>sys.stderr,"Connecter: EXTEND: my dict",self.EXTEND_HANDSHAKE_M_DICT # BarterCast if config['overlay']: self.overlay_bridge = OverlayThreadingBridge.getInstance() else: self.overlay_bridge = None
def __init__(self, make_upload, downloader, choker, numpieces, piece_size, totalup, config, ratelimiter, merkle_torrent, sched=None, coordinator=None, helper=None, mylistenport=None, use_g2g=False, infohash=None, tracker=None): self.downloader = downloader self.make_upload = make_upload self.choker = choker self.numpieces = numpieces self.piece_size = piece_size self.config = config self.ratelimiter = ratelimiter self.rate_capped = False self.sched = sched self.totalup = totalup self.rate_capped = False self.connections = {} self.external_connection_made = 0 self.merkle_torrent = merkle_torrent self.use_g2g = use_g2g # 2fastbt_ self.coordinator = coordinator self.helper = helper self.round = 0 self.mylistenport = mylistenport self.infohash = infohash self.tracker = tracker try: (scheme, netloc, path, pars, query, fragment) = urlparse.urlparse(self.tracker) host = netloc.split(':')[0] self.tracker_ip = socket.getaddrinfo(host, None)[0][4][0] except: print_exc() self.tracker_ip = None print >> sys.stderr, "Connecter: live: source/tracker is", self.tracker_ip self.overlay_enabled = 0 if self.config['overlay']: self.overlay_enabled = True if DEBUG: if self.overlay_enabled: print >> sys.stderr, "connecter: Enabling overlay" else: print >> sys.stderr, "connecter: Disabling overlay" if DEBUG: if self.overlay_enabled: print >> sys.stderr, "connecter: Enabling overlay" else: print >> sys.stderr, "connecter: Disabling overlay" self.ut_pex_enabled = 0 if 'ut_pex_max_addrs_from_peer' in self.config: self.ut_pex_max_addrs_from_peer = self.config[ 'ut_pex_max_addrs_from_peer'] self.ut_pex_enabled = self.ut_pex_max_addrs_from_peer > 0 self.ut_pex_previous_conns = [ ] # last value of 'added' field for all peers if DEBUG_UT_PEX: if self.ut_pex_enabled: print >> sys.stderr, "connecter: Enabling uTorrent PEX", self.ut_pex_max_addrs_from_peer else: print >> sys.stderr, "connecter: Disabling uTorrent PEX" # The set of messages we support. Note that the msg ID is an int not a byte in # this dict. self.EXTEND_HANDSHAKE_M_DICT = {} if self.overlay_enabled: # Say in the EXTEND handshake we support the overlay-swarm ext. d = {EXTEND_MSG_OVERLAYSWARM: ord(CHALLENGE)} self.EXTEND_HANDSHAKE_M_DICT.update(d) if self.ut_pex_enabled: # Say in the EXTEND handshake we support uTorrent's peer exchange ext. d = {EXTEND_MSG_UTORRENT_PEX: ord(EXTEND_MSG_UTORRENT_PEX_ID)} self.EXTEND_HANDSHAKE_M_DICT.update(d) self.sched(self.ut_pex_callback, 6) if self.use_g2g: # Say in the EXTEND handshake we want to do G2G. d = {EXTEND_MSG_G2G_V2: ord(G2G_PIECE_XFER)} self.EXTEND_HANDSHAKE_M_DICT.update(d) self.sched(self.g2g_callback, G2G_CALLBACK_INTERVAL) # LIVEHACK livekey = EXTEND_MSG_LIVE_PREFIX + str(CURRENT_LIVE_VERSION) d = {livekey: ord(LIVE_FAKE_MESSAGE_ID)} self.EXTEND_HANDSHAKE_M_DICT.update(d) print >> sys.stderr, "Connecter: EXTEND: my dict", self.EXTEND_HANDSHAKE_M_DICT # BarterCast if config['overlay']: self.overlay_bridge = OverlayThreadingBridge.getInstance() else: self.overlay_bridge = None
print >> sys.stderr, "superpeer: config is", config if config['overlaylogpostfix'] == '': config['overlaylogpostfix'] = 'sp' + str(config['port']) + '.log' # # Currently we use an in-memory database for superpeers. # SQLite supports only per-thread in memory databases. # As our Session initialization is currently setup, the MainThread # creates the DB and adds the superpeer entries, and the OverlayThread # does most DB operations. So two threads accessing the DB. # # To work around this I start the Session using the OverlayThread. # Dirty, but a simple solution. # overlay_bridge = OverlayThreadingBridge.getInstance() overlay_bridge.add_task(olthread_start_session, 0) # # NetworkThread and OverlayThread will now do their work. The MainThread # running this here code should wait indefinitely to avoid exiting the # process. # try: while True: # time.sleep(sys.maxint) has "issues" on 64bit architectures; divide it # by some value (2048) to solve problem time.sleep(sys.maxint / 2048) except: print_exc()
def setUp(self): print >>sys.stderr,"test: TestOverlayThreadingBridge.setUp()" self.config_path = tempfile.mkdtemp() config = {} config['state_dir'] = self.config_path config['torrent_collecting_dir'] = self.config_path config['install_dir'] = os.path.join('..','..') config['peer_icon_path'] = os.path.join(self.config_path,'peer_icons') config['superpeer'] = False sqlitecachedb.init(config, self.rawserver_fatalerrorfunc) secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() overbridge1 = OverlayThreadingBridge() overbridge1.register_bridge(secover1,None) overbridge1.resetSingleton() overbridge2 = OverlayThreadingBridge() overbridge2.register_bridge(secover2,None) overbridge2.resetSingleton() self.peer1 = Peer(self,1234,overbridge1) self.peer2 = Peer(self,5678,overbridge2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid) print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid) time.sleep(2) # let server threads start