コード例 #1
0
    def unload_community(self):
        if __debug__: dprint()
        super(EffortCommunity, self).unload_community()

        # unsubscribe from events.  22/03/13 Boudewijn: Dispersy is not allowed to call swift
        # because it may cause database locks if people incorrectly keep the session locked.
        temp_thread = Callback()
        temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", False, self.i2ithread_channel_close))
        temp_thread.register(temp_thread.stop)
        temp_thread.start("Temporary-EffortCommunity")

        # cancel outstanding pings
        for ping_candidate in self._slope.itervalues():
            self._dispersy.callback.unregister(ping_candidate.callback_id)
        self._slope = {}

        # store all cached observations
        self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
                                   [(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])

        # update all up and download values
        self.download_state_callback([], closing=True)

        # store all cached bandwidth guesses
        self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
                                   [(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
コード例 #2
0
    def __init__(self, master, swift_process):
        # original walker callbacks (will be set during super(...).__init__)
        self._original_on_introduction_request = None
        self._original_on_introduction_response = None

        super(EffortCommunity, self).__init__(master)

        # _SWIFT is a SwiftProcess instance (allowing us to schedule CLOSE_EVENT callbacks)
        self._swift = swift_process
        # subscribe to events.  22/03/13 Boudewijn: Dispersy is not allowed to call swift because it
        # may cause database locks if people incorrectly keep the session locked.
        temp_thread = Callback()
        temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", True, self.i2ithread_channel_close))
        temp_thread.register(temp_thread.stop)
        temp_thread.start("Temporary-EffortCommunity")

        # _DATABASE stores all direct observations and indirect hearsay
        self._database = EffortDatabase.get_instance(self._dispersy)

        # _OBSERVATIONS cache (reduce _DATABASE access)
        self._observations_length = 512
        self._observations = OrderedDict()

        # _bandwidth_guesses cache (reduce _DATABASE access)
        self._bandwidth_guesses_length = 512
        self._bandwidth_guesses = OrderedDict()

        # _DOWNLOAD_STATES contains all peers that are currently downloading.  when we determine
        # that a peer is missing, we will update its bandwidth statistics
        self._download_states = dict()
        self._swift_raw_bytes_up = 0
        self._swift_raw_bytes_down = 0

        # _SLOPE contains the promising members as Member:RecordCandidate
        self._slope_length = 10
        self._slope = {}

        # _SIGNATURE_COUNT is the number of members that will be asked to sign
        self._signature_count = 5

        # simple statistics
        self._statistic_incoming_signature_request_success = 0
        self._statistic_outgoing_signature_request = 0
        self._statistic_outgoing_signature_request_success = 0
        self._statistic_outgoing_signature_request_timeout = 0
        self._statistic_member_ordering_fail = 0
        self._statistic_initial_timestamp_fail = 0
        self._statistic_cycle_fail = 0

        self._has_been_killed = False

        # wait till next time we can create records with the candidates on our slope
        self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_create_records))
        self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_push_statistics))
        self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_cleanup_database))
コード例 #3
0
    def start_dispersy(self):
        msg("Starting dispersy")
        # We need to import the stuff _AFTER_ configuring the logging stuff.
        from Tribler.dispersy.callback import Callback
        from Tribler.dispersy.dispersy import Dispersy
        from Tribler.dispersy.endpoint import StandaloneEndpoint

        self._dispersy = Dispersy(
            Callback("Dispersy"),
            StandaloneEndpoint(int(self.my_id) + 12000, '0.0.0.0'), u'.',
            self._database_file)
        self._dispersy.statistics.enable_debug_statistics(True)

        if self._strict:

            def exception_handler(exception, fatal):
                msg("An exception occurred. Quitting because we are running with --strict enabled."
                    )
                print "Exception was:"

                try:
                    raise exception
                except:
                    from traceback import print_exc
                    print_exc()

                # Set Dispersy's exit status to error
                self._dispersy_exit_status = 1
                # Stop the experiment
                reactor.callLater(1, self.stop)

                return True

            self._dispersy.callback.attach_exception_handler(exception_handler)

        self._dispersy.start()

        # low (NID_sect233k1) isn't actually that low, switching to 160bits as this is comparable to rsa 1024
        # http://www.nsa.gov/business/programs/elliptic_curve.shtml
        # speed difference when signing/verifying 100 items
        # NID_sect233k1 signing took 0.171 verify took 0.35 totals 0.521
        # NID_secp160k1 signing took 0.04 verify took 0.04 totals 0.08
        self._my_member = self._dispersy.callback.call(
            self._dispersy.get_new_member, (u"NID_secp160k1", ))
        self._master_member = self._dispersy.callback.call(
            self._dispersy.get_member, (self.master_key, ))

        self._dispersy.callback.register(self._do_log)
        msg("Finished starting dispersy")
コード例 #4
0
    parser.add_argument("-l", "--listen", metavar="IP", help='local listen ip')
    parser.add_argument("-i", "--tracker-ip", default="127.0.0.1", metavar="IP", help='Dispersy tracker IP')
    parser.add_argument("-p", "--tracker-port", type=int, default=6421, metavar="PORT", help='Dispersy tracker port')
    args = parser.parse_args()

    from Tribler.dispersy.script import ScriptBase
    from Tribler.dispersy.member import Member
    from Tribler.dispersy.callback import Callback
    from Tribler.dispersy.dispersy import Dispersy
    from Tribler.dispersy.endpoint import StandaloneEndpoint

    with open(BOOTSTRAPTRIBLER_FILE, "w") as f:
        print >> f, args.tracker_ip, args.tracker_port

    # start in-memory Dispersy
    callback = Callback('Dispersy')
    if args.listen != None:
        endpoint = StandaloneEndpoint(randint(10000, 20000), args.listen)
    else:
        endpoint = StandaloneEndpoint(randint(10000, 20000))
    dispersy = Dispersy(callback, endpoint, u'.')
    dispersy.start()
    logger.info("Dispersy is listening on " + repr(dispersy.lan_address))

    callback.call(create_bc3_community, (dispersy,))

    # wait
    # TODO(vladum): Wait until community is created (or just some time).
    # try:
    #     while callback.is_running:
    #         time.sleep(5.0)
コード例 #5
0
ファイル: LaunchManyCore.py プロジェクト: Anbcorp/tribler
    def register(self, session, sesslock):
        if not self.registered:
            self.registered = True

            self.session = session
            self.sesslock = sesslock

            self.downloads = {}
            config = session.sessconfig  # Should be safe at startup

            self.upnp_ports = []

            # Orig
            self.sessdoneflag = Event()

            self.rawserver = RawServer(self.sessdoneflag,
                                       config['timeout_check_interval'],
                                       config['timeout'],
                                       ipv6_enable=config['ipv6_enabled'],
                                       failfunc=self.rawserver_fatalerrorfunc,
                                       errorfunc=self.rawserver_nonfatalerrorfunc)
            self.rawserver.add_task(self.rawserver_keepalive, 1)
            self.listen_port = config['minport']
            self.shutdownstarttime = None

            self.multihandler = MultiHandler(self.rawserver, self.sessdoneflag)

            # SWIFTPROC
            swift_exists = config['swiftproc'] and (os.path.exists(config['swiftpath']) or os.path.exists(config['swiftpath'] + '.exe'))
            if swift_exists:
                from Tribler.Core.Swift.SwiftProcessMgr import SwiftProcessMgr

                self.spm = SwiftProcessMgr(config['swiftpath'], config['swiftcmdlistenport'], config['swiftdlsperproc'], self.session.get_swift_tunnel_listen_port(), self.sesslock)
                try:
                    self.swift_process = self.spm.get_or_create_sp(self.session.get_swift_working_dir(), self.session.get_torrent_collecting_dir(), self.session.get_swift_tunnel_listen_port(), self.session.get_swift_tunnel_httpgw_listen_port(), self.session.get_swift_tunnel_cmdgw_listen_port())
                    self.upnp_ports.append((self.session.get_swift_tunnel_listen_port(), 'UDP'))

                except OSError:
                    # could not find/run swift
                    print >> sys.stderr, "lmc: could not start a swift process"

            else:
                self.spm = None
                self.swift_process = None

            # Dispersy
            self.session.dispersy_member = None
            if config['dispersy']:
                from Tribler.dispersy.callback import Callback
                from Tribler.dispersy.dispersy import Dispersy
                from Tribler.dispersy.endpoint import RawserverEndpoint, TunnelEndpoint
                from Tribler.dispersy.community import HardKilledCommunity

                # set communication endpoint
                if config['dispersy-tunnel-over-swift'] and self.swift_process:
                    endpoint = TunnelEndpoint(self.swift_process)
                else:
                    endpoint = RawserverEndpoint(self.rawserver, config['dispersy_port'])

                callback = Callback("Dispersy")  # WARNING NAME SIGNIFICANT
                working_directory = unicode(config['state_dir'])

                self.dispersy = Dispersy(callback, endpoint, working_directory)

                # TODO: see if we can postpone dispersy.start to improve GUI responsiveness.
                # However, for now we must start self.dispersy.callback before running
                # try_register(nocachedb, self.database_thread)!

                self.dispersy.start()

                print >> sys.stderr, "lmc: Dispersy is listening on port", self.dispersy.wan_address[1], "using", endpoint
                self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))

                self.dispersy.callback.call(self.dispersy.define_auto_load, args=(HardKilledCommunity,), kargs={'load': True})

                # notify dispersy finished loading
                self.session.uch.notify(NTFY_DISPERSY, NTFY_STARTED, None)

                from Tribler.Core.permid import read_keypair
                from Tribler.dispersy.crypto import ec_to_public_bin, ec_to_private_bin
                keypair = read_keypair(self.session.get_permid_keypair_filename())
                self.session.dispersy_member = callback.call(self.dispersy.get_member, (ec_to_public_bin(keypair), ec_to_private_bin(keypair)))

                self.database_thread = callback
            else:
                class FakeCallback():
                    def __init__(self):
                        from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
                        self.queue = TimedTaskQueue("FakeCallback")

                    def register(self, call, args=(), kargs=None, delay=0.0, priority=0, id_=u"", callback=None, callback_args=(), callback_kargs=None, include_id=False):
                        def do_task():
                            if kargs:
                                call(*args, **kargs)
                            else:
                                call(*args)

                            if callback:
                                if callback_kargs:
                                    callback(*callback_args, **callback_kargs)
                                else:
                                    callback(*callback_args)
                        self.queue.add_task(do_task, t=delay)

                    def shutdown(self, immediately=False):
                        self.queue.shutdown(immediately)

                self.database_thread = FakeCallback()

            if config['megacache']:
                import Tribler.Core.CacheDB.cachedb as cachedb
                from Tribler.Core.CacheDB.SqliteCacheDBHandler import PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler, VoteCastDBHandler, ChannelCastDBHandler, NetworkBuzzDBHandler, UserEventLogDBHandler
                from Tribler.Category.Category import Category
                from Tribler.Core.Tag.Extraction import TermExtraction
                from Tribler.Core.CacheDB.sqlitecachedb import try_register

                if DEBUG:
                    print >> sys.stderr, 'tlm: Reading Session state from', config['state_dir']

                nocachedb = cachedb.init(config, self.rawserver_fatalerrorfunc)
                try_register(nocachedb, self.database_thread)

                self.cat = Category.getInstance(config['install_dir'])
                self.term = TermExtraction.getInstance(config['install_dir'])

                self.peer_db = PeerDBHandler.getInstance()
                self.peer_db.registerConnectionUpdater(self.session)

                self.torrent_db = TorrentDBHandler.getInstance()
                self.torrent_db.register(os.path.abspath(config['torrent_collecting_dir']))
                self.mypref_db = MyPreferenceDBHandler.getInstance()
                self.votecast_db = VoteCastDBHandler.getInstance()
                self.votecast_db.registerSession(self.session)
                self.channelcast_db = ChannelCastDBHandler.getInstance()
                self.channelcast_db.registerSession(self.session)
                self.nb_db = NetworkBuzzDBHandler.getInstance()
                self.ue_db = UserEventLogDBHandler.getInstance()

                if self.dispersy:
                    self.dispersy.database.attach_commit_callback(self.channelcast_db._db.commitNow)
            else:
                config['torrent_checking'] = 0

            self.rtorrent_handler = None
            if config['torrent_collecting']:
                from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
                self.rtorrent_handler = RemoteTorrentHandler()
コード例 #6
0
    def register(self, session, sesslock):
        if not self.registered:
            self.registered = True

            self.session = session
            self.sesslock = sesslock

            self.downloads = {}
            config = session.sessconfig  # Should be safe at startup

            self.upnp_ports = []

            # Orig
            self.sessdoneflag = Event()

            self.rawserver = RawServer(
                self.sessdoneflag,
                config['timeout_check_interval'],
                config['timeout'],
                ipv6_enable=config['ipv6_enabled'],
                failfunc=self.rawserver_fatalerrorfunc,
                errorfunc=self.rawserver_nonfatalerrorfunc)
            self.rawserver.add_task(self.rawserver_keepalive, 1)
            self.listen_port = config['minport']
            self.shutdownstarttime = None

            self.multihandler = MultiHandler(self.rawserver, self.sessdoneflag)

            # SWIFTPROC
            swift_exists = config['swiftproc'] and (
                os.path.exists(config['swiftpath'])
                or os.path.exists(config['swiftpath'] + '.exe'))
            if swift_exists:
                from Tribler.Core.Swift.SwiftProcessMgr import SwiftProcessMgr

                self.spm = SwiftProcessMgr(
                    config['swiftpath'], config['swiftcmdlistenport'],
                    config['swiftdlsperproc'],
                    self.session.get_swift_tunnel_listen_port(), self.sesslock)
                try:
                    self.swift_process = self.spm.get_or_create_sp(
                        self.session.get_swift_working_dir(),
                        self.session.get_torrent_collecting_dir(),
                        self.session.get_swift_tunnel_listen_port(),
                        self.session.get_swift_tunnel_httpgw_listen_port(),
                        self.session.get_swift_tunnel_cmdgw_listen_port())
                    self.upnp_ports.append(
                        (self.session.get_swift_tunnel_listen_port(), 'UDP'))

                except OSError:
                    # could not find/run swift
                    print >> sys.stderr, "lmc: could not start a swift process"

            else:
                self.spm = None
                self.swift_process = None

            # Dispersy
            self.session.dispersy_member = None
            if config['dispersy']:
                from Tribler.dispersy.callback import Callback
                from Tribler.dispersy.dispersy import Dispersy
                from Tribler.dispersy.endpoint import RawserverEndpoint, TunnelEndpoint
                from Tribler.dispersy.community import HardKilledCommunity

                # set communication endpoint
                if config['dispersy-tunnel-over-swift'] and self.swift_process:
                    endpoint = TunnelEndpoint(self.swift_process)
                else:
                    endpoint = RawserverEndpoint(self.rawserver,
                                                 config['dispersy_port'])

                callback = Callback("Dispersy")  # WARNING NAME SIGNIFICANT
                working_directory = unicode(config['state_dir'])

                self.dispersy = Dispersy(callback, endpoint, working_directory)

                # TODO: see if we can postpone dispersy.start to improve GUI responsiveness.
                # However, for now we must start self.dispersy.callback before running
                # try_register(nocachedb, self.database_thread)!

                self.dispersy.start()

                print >> sys.stderr, "lmc: Dispersy is listening on port", self.dispersy.wan_address[
                    1], "using", endpoint
                self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))

                self.dispersy.callback.call(self.dispersy.define_auto_load,
                                            args=(HardKilledCommunity, ),
                                            kargs={'load': True})

                # notify dispersy finished loading
                self.session.uch.notify(NTFY_DISPERSY, NTFY_STARTED, None)

                from Tribler.Core.permid import read_keypair
                from Tribler.dispersy.crypto import ec_to_public_bin, ec_to_private_bin
                keypair = read_keypair(
                    self.session.get_permid_keypair_filename())
                self.session.dispersy_member = callback.call(
                    self.dispersy.get_member,
                    (ec_to_public_bin(keypair), ec_to_private_bin(keypair)))

                self.database_thread = callback
            else:

                class FakeCallback():
                    def __init__(self):
                        from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
                        self.queue = TimedTaskQueue("FakeCallback")

                    def register(self,
                                 call,
                                 args=(),
                                 kargs=None,
                                 delay=0.0,
                                 priority=0,
                                 id_=u"",
                                 callback=None,
                                 callback_args=(),
                                 callback_kargs=None,
                                 include_id=False):
                        def do_task():
                            if kargs:
                                call(*args, **kargs)
                            else:
                                call(*args)

                            if callback:
                                if callback_kargs:
                                    callback(*callback_args, **callback_kargs)
                                else:
                                    callback(*callback_args)

                        self.queue.add_task(do_task, t=delay)

                    def shutdown(self, immediately=False):
                        self.queue.shutdown(immediately)

                self.database_thread = FakeCallback()

            if config['megacache']:
                import Tribler.Core.CacheDB.cachedb as cachedb
                from Tribler.Core.CacheDB.SqliteCacheDBHandler import PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler, VoteCastDBHandler, ChannelCastDBHandler, NetworkBuzzDBHandler, UserEventLogDBHandler
                from Tribler.Category.Category import Category
                from Tribler.Core.Tag.Extraction import TermExtraction
                from Tribler.Core.CacheDB.sqlitecachedb import try_register

                if DEBUG:
                    print >> sys.stderr, 'tlm: Reading Session state from', config[
                        'state_dir']

                nocachedb = cachedb.init(config, self.rawserver_fatalerrorfunc)
                try_register(nocachedb, self.database_thread)

                self.cat = Category.getInstance(config['install_dir'])
                self.term = TermExtraction.getInstance(config['install_dir'])

                self.peer_db = PeerDBHandler.getInstance()
                self.peer_db.registerConnectionUpdater(self.session)

                self.torrent_db = TorrentDBHandler.getInstance()
                self.torrent_db.register(
                    os.path.abspath(config['torrent_collecting_dir']))
                self.mypref_db = MyPreferenceDBHandler.getInstance()
                self.votecast_db = VoteCastDBHandler.getInstance()
                self.votecast_db.registerSession(self.session)
                self.channelcast_db = ChannelCastDBHandler.getInstance()
                self.channelcast_db.registerSession(self.session)
                self.nb_db = NetworkBuzzDBHandler.getInstance()
                self.ue_db = UserEventLogDBHandler.getInstance()

                if self.dispersy:
                    self.dispersy.database.attach_commit_callback(
                        self.channelcast_db._db.commitNow)
            else:
                config['torrent_checking'] = 0

            self.rtorrent_handler = None
            if config['torrent_collecting']:
                from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
                self.rtorrent_handler = RemoteTorrentHandler()