def unload_community(self):
if __debug__: dprint()
super(EffortCommunity, self).unload_community()
# unsubscribe from events. 22/03/13 Boudewijn: Dispersy is not allowed to call swift
# because it may cause database locks if people incorrectly keep the session locked.
temp_thread = Callback()
temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", False, self.i2ithread_channel_close))
temp_thread.register(temp_thread.stop)
temp_thread.start("Temporary-EffortCommunity")
# cancel outstanding pings
for ping_candidate in self._slope.itervalues():
self._dispersy.callback.unregister(ping_candidate.callback_id)
self._slope = {}
# store all cached observations
self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
[(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])
# update all up and download values
self.download_state_callback([], closing=True)
# store all cached bandwidth guesses
self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
[(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
def __init__(self, master, swift_process):
# original walker callbacks (will be set during super(...).__init__)
self._original_on_introduction_request = None
self._original_on_introduction_response = None
super(EffortCommunity, self).__init__(master)
# _SWIFT is a SwiftProcess instance (allowing us to schedule CLOSE_EVENT callbacks)
self._swift = swift_process
# subscribe to events. 22/03/13 Boudewijn: Dispersy is not allowed to call swift because it
# may cause database locks if people incorrectly keep the session locked.
temp_thread = Callback()
temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", True, self.i2ithread_channel_close))
temp_thread.register(temp_thread.stop)
temp_thread.start("Temporary-EffortCommunity")
# _DATABASE stores all direct observations and indirect hearsay
self._database = EffortDatabase.get_instance(self._dispersy)
# _OBSERVATIONS cache (reduce _DATABASE access)
self._observations_length = 512
self._observations = OrderedDict()
# _bandwidth_guesses cache (reduce _DATABASE access)
self._bandwidth_guesses_length = 512
self._bandwidth_guesses = OrderedDict()
# _DOWNLOAD_STATES contains all peers that are currently downloading. when we determine
# that a peer is missing, we will update its bandwidth statistics
self._download_states = dict()
self._swift_raw_bytes_up = 0
self._swift_raw_bytes_down = 0
# _SLOPE contains the promising members as Member:RecordCandidate
self._slope_length = 10
self._slope = {}
# _SIGNATURE_COUNT is the number of members that will be asked to sign
self._signature_count = 5
# simple statistics
self._statistic_incoming_signature_request_success = 0
self._statistic_outgoing_signature_request = 0
self._statistic_outgoing_signature_request_success = 0
self._statistic_outgoing_signature_request_timeout = 0
self._statistic_member_ordering_fail = 0
self._statistic_initial_timestamp_fail = 0
self._statistic_cycle_fail = 0
self._has_been_killed = False
# wait till next time we can create records with the candidates on our slope
self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_create_records))
self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_push_statistics))
self._pending_callbacks.append(self._dispersy.callback.register(self._periodically_cleanup_database))
def start_dispersy(self):
msg("Starting dispersy")
# We need to import the stuff _AFTER_ configuring the logging stuff.
from Tribler.dispersy.callback import Callback
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import StandaloneEndpoint
self._dispersy = Dispersy(
Callback("Dispersy"),
StandaloneEndpoint(int(self.my_id) + 12000, '0.0.0.0'), u'.',
self._database_file)
self._dispersy.statistics.enable_debug_statistics(True)
if self._strict:
def exception_handler(exception, fatal):
msg("An exception occurred. Quitting because we are running with --strict enabled."
)
print "Exception was:"
try:
raise exception
except:
from traceback import print_exc
print_exc()
# Set Dispersy's exit status to error
self._dispersy_exit_status = 1
# Stop the experiment
reactor.callLater(1, self.stop)
return True
self._dispersy.callback.attach_exception_handler(exception_handler)
self._dispersy.start()
# low (NID_sect233k1) isn't actually that low, switching to 160bits as this is comparable to rsa 1024
# http://www.nsa.gov/business/programs/elliptic_curve.shtml
# speed difference when signing/verifying 100 items
# NID_sect233k1 signing took 0.171 verify took 0.35 totals 0.521
# NID_secp160k1 signing took 0.04 verify took 0.04 totals 0.08
self._my_member = self._dispersy.callback.call(
self._dispersy.get_new_member, (u"NID_secp160k1", ))
self._master_member = self._dispersy.callback.call(
self._dispersy.get_member, (self.master_key, ))
self._dispersy.callback.register(self._do_log)
msg("Finished starting dispersy")
parser.add_argument("-l", "--listen", metavar="IP", help='local listen ip')
parser.add_argument("-i", "--tracker-ip", default="127.0.0.1", metavar="IP", help='Dispersy tracker IP')
parser.add_argument("-p", "--tracker-port", type=int, default=6421, metavar="PORT", help='Dispersy tracker port')
args = parser.parse_args()
from Tribler.dispersy.script import ScriptBase
from Tribler.dispersy.member import Member
from Tribler.dispersy.callback import Callback
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import StandaloneEndpoint
with open(BOOTSTRAPTRIBLER_FILE, "w") as f:
print >> f, args.tracker_ip, args.tracker_port
# start in-memory Dispersy
callback = Callback('Dispersy')
if args.listen != None:
endpoint = StandaloneEndpoint(randint(10000, 20000), args.listen)
else:
endpoint = StandaloneEndpoint(randint(10000, 20000))
dispersy = Dispersy(callback, endpoint, u'.')
dispersy.start()
logger.info("Dispersy is listening on " + repr(dispersy.lan_address))
callback.call(create_bc3_community, (dispersy,))
# wait
# TODO(vladum): Wait until community is created (or just some time).
# try:
# while callback.is_running:
# time.sleep(5.0)
def register(self, session, sesslock):
if not self.registered:
self.registered = True
self.session = session
self.sesslock = sesslock
self.downloads = {}
config = session.sessconfig # Should be safe at startup
self.upnp_ports = []
# Orig
self.sessdoneflag = Event()
self.rawserver = RawServer(self.sessdoneflag,
config['timeout_check_interval'],
config['timeout'],
ipv6_enable=config['ipv6_enabled'],
failfunc=self.rawserver_fatalerrorfunc,
errorfunc=self.rawserver_nonfatalerrorfunc)
self.rawserver.add_task(self.rawserver_keepalive, 1)
self.listen_port = config['minport']
self.shutdownstarttime = None
self.multihandler = MultiHandler(self.rawserver, self.sessdoneflag)
# SWIFTPROC
swift_exists = config['swiftproc'] and (os.path.exists(config['swiftpath']) or os.path.exists(config['swiftpath'] + '.exe'))
if swift_exists:
from Tribler.Core.Swift.SwiftProcessMgr import SwiftProcessMgr
self.spm = SwiftProcessMgr(config['swiftpath'], config['swiftcmdlistenport'], config['swiftdlsperproc'], self.session.get_swift_tunnel_listen_port(), self.sesslock)
try:
self.swift_process = self.spm.get_or_create_sp(self.session.get_swift_working_dir(), self.session.get_torrent_collecting_dir(), self.session.get_swift_tunnel_listen_port(), self.session.get_swift_tunnel_httpgw_listen_port(), self.session.get_swift_tunnel_cmdgw_listen_port())
self.upnp_ports.append((self.session.get_swift_tunnel_listen_port(), 'UDP'))
except OSError:
# could not find/run swift
print >> sys.stderr, "lmc: could not start a swift process"
else:
self.spm = None
self.swift_process = None
# Dispersy
self.session.dispersy_member = None
if config['dispersy']:
from Tribler.dispersy.callback import Callback
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import RawserverEndpoint, TunnelEndpoint
from Tribler.dispersy.community import HardKilledCommunity
# set communication endpoint
if config['dispersy-tunnel-over-swift'] and self.swift_process:
endpoint = TunnelEndpoint(self.swift_process)
else:
endpoint = RawserverEndpoint(self.rawserver, config['dispersy_port'])
callback = Callback("Dispersy") # WARNING NAME SIGNIFICANT
working_directory = unicode(config['state_dir'])
self.dispersy = Dispersy(callback, endpoint, working_directory)
# TODO: see if we can postpone dispersy.start to improve GUI responsiveness.
# However, for now we must start self.dispersy.callback before running
# try_register(nocachedb, self.database_thread)!
self.dispersy.start()
print >> sys.stderr, "lmc: Dispersy is listening on port", self.dispersy.wan_address[1], "using", endpoint
self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))
self.dispersy.callback.call(self.dispersy.define_auto_load, args=(HardKilledCommunity,), kargs={'load': True})
# notify dispersy finished loading
self.session.uch.notify(NTFY_DISPERSY, NTFY_STARTED, None)
from Tribler.Core.permid import read_keypair
from Tribler.dispersy.crypto import ec_to_public_bin, ec_to_private_bin
keypair = read_keypair(self.session.get_permid_keypair_filename())
self.session.dispersy_member = callback.call(self.dispersy.get_member, (ec_to_public_bin(keypair), ec_to_private_bin(keypair)))
self.database_thread = callback
else:
class FakeCallback():
def __init__(self):
from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
self.queue = TimedTaskQueue("FakeCallback")
def register(self, call, args=(), kargs=None, delay=0.0, priority=0, id_=u"", callback=None, callback_args=(), callback_kargs=None, include_id=False):
def do_task():
if kargs:
call(*args, **kargs)
else:
call(*args)
if callback:
if callback_kargs:
callback(*callback_args, **callback_kargs)
else:
callback(*callback_args)
self.queue.add_task(do_task, t=delay)
def shutdown(self, immediately=False):
self.queue.shutdown(immediately)
self.database_thread = FakeCallback()
if config['megacache']:
import Tribler.Core.CacheDB.cachedb as cachedb
from Tribler.Core.CacheDB.SqliteCacheDBHandler import PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler, VoteCastDBHandler, ChannelCastDBHandler, NetworkBuzzDBHandler, UserEventLogDBHandler
from Tribler.Category.Category import Category
from Tribler.Core.Tag.Extraction import TermExtraction
from Tribler.Core.CacheDB.sqlitecachedb import try_register
if DEBUG:
print >> sys.stderr, 'tlm: Reading Session state from', config['state_dir']
nocachedb = cachedb.init(config, self.rawserver_fatalerrorfunc)
try_register(nocachedb, self.database_thread)
self.cat = Category.getInstance(config['install_dir'])
self.term = TermExtraction.getInstance(config['install_dir'])
self.peer_db = PeerDBHandler.getInstance()
self.peer_db.registerConnectionUpdater(self.session)
self.torrent_db = TorrentDBHandler.getInstance()
self.torrent_db.register(os.path.abspath(config['torrent_collecting_dir']))
self.mypref_db = MyPreferenceDBHandler.getInstance()
self.votecast_db = VoteCastDBHandler.getInstance()
self.votecast_db.registerSession(self.session)
self.channelcast_db = ChannelCastDBHandler.getInstance()
self.channelcast_db.registerSession(self.session)
self.nb_db = NetworkBuzzDBHandler.getInstance()
self.ue_db = UserEventLogDBHandler.getInstance()
if self.dispersy:
self.dispersy.database.attach_commit_callback(self.channelcast_db._db.commitNow)
else:
config['torrent_checking'] = 0
self.rtorrent_handler = None
if config['torrent_collecting']:
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
self.rtorrent_handler = RemoteTorrentHandler()
def register(self, session, sesslock):
if not self.registered:
self.registered = True
self.session = session
self.sesslock = sesslock
self.downloads = {}
config = session.sessconfig # Should be safe at startup
self.upnp_ports = []
# Orig
self.sessdoneflag = Event()
self.rawserver = RawServer(
self.sessdoneflag,
config['timeout_check_interval'],
config['timeout'],
ipv6_enable=config['ipv6_enabled'],
failfunc=self.rawserver_fatalerrorfunc,
errorfunc=self.rawserver_nonfatalerrorfunc)
self.rawserver.add_task(self.rawserver_keepalive, 1)
self.listen_port = config['minport']
self.shutdownstarttime = None
self.multihandler = MultiHandler(self.rawserver, self.sessdoneflag)
# SWIFTPROC
swift_exists = config['swiftproc'] and (
os.path.exists(config['swiftpath'])
or os.path.exists(config['swiftpath'] + '.exe'))
if swift_exists:
from Tribler.Core.Swift.SwiftProcessMgr import SwiftProcessMgr
self.spm = SwiftProcessMgr(
config['swiftpath'], config['swiftcmdlistenport'],
config['swiftdlsperproc'],
self.session.get_swift_tunnel_listen_port(), self.sesslock)
try:
self.swift_process = self.spm.get_or_create_sp(
self.session.get_swift_working_dir(),
self.session.get_torrent_collecting_dir(),
self.session.get_swift_tunnel_listen_port(),
self.session.get_swift_tunnel_httpgw_listen_port(),
self.session.get_swift_tunnel_cmdgw_listen_port())
self.upnp_ports.append(
(self.session.get_swift_tunnel_listen_port(), 'UDP'))
except OSError:
# could not find/run swift
print >> sys.stderr, "lmc: could not start a swift process"
else:
self.spm = None
self.swift_process = None
# Dispersy
self.session.dispersy_member = None
if config['dispersy']:
from Tribler.dispersy.callback import Callback
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import RawserverEndpoint, TunnelEndpoint
from Tribler.dispersy.community import HardKilledCommunity
# set communication endpoint
if config['dispersy-tunnel-over-swift'] and self.swift_process:
endpoint = TunnelEndpoint(self.swift_process)
else:
endpoint = RawserverEndpoint(self.rawserver,
config['dispersy_port'])
callback = Callback("Dispersy") # WARNING NAME SIGNIFICANT
working_directory = unicode(config['state_dir'])
self.dispersy = Dispersy(callback, endpoint, working_directory)
# TODO: see if we can postpone dispersy.start to improve GUI responsiveness.
# However, for now we must start self.dispersy.callback before running
# try_register(nocachedb, self.database_thread)!
self.dispersy.start()
print >> sys.stderr, "lmc: Dispersy is listening on port", self.dispersy.wan_address[
1], "using", endpoint
self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))
self.dispersy.callback.call(self.dispersy.define_auto_load,
args=(HardKilledCommunity, ),
kargs={'load': True})
# notify dispersy finished loading
self.session.uch.notify(NTFY_DISPERSY, NTFY_STARTED, None)
from Tribler.Core.permid import read_keypair
from Tribler.dispersy.crypto import ec_to_public_bin, ec_to_private_bin
keypair = read_keypair(
self.session.get_permid_keypair_filename())
self.session.dispersy_member = callback.call(
self.dispersy.get_member,
(ec_to_public_bin(keypair), ec_to_private_bin(keypair)))
self.database_thread = callback
else:
class FakeCallback():
def __init__(self):
from Tribler.Utilities.TimedTaskQueue import TimedTaskQueue
self.queue = TimedTaskQueue("FakeCallback")
def register(self,
call,
args=(),
kargs=None,
delay=0.0,
priority=0,
id_=u"",
callback=None,
callback_args=(),
callback_kargs=None,
include_id=False):
def do_task():
if kargs:
call(*args, **kargs)
else:
call(*args)
if callback:
if callback_kargs:
callback(*callback_args, **callback_kargs)
else:
callback(*callback_args)
self.queue.add_task(do_task, t=delay)
def shutdown(self, immediately=False):
self.queue.shutdown(immediately)
self.database_thread = FakeCallback()
if config['megacache']:
import Tribler.Core.CacheDB.cachedb as cachedb
from Tribler.Core.CacheDB.SqliteCacheDBHandler import PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler, VoteCastDBHandler, ChannelCastDBHandler, NetworkBuzzDBHandler, UserEventLogDBHandler
from Tribler.Category.Category import Category
from Tribler.Core.Tag.Extraction import TermExtraction
from Tribler.Core.CacheDB.sqlitecachedb import try_register
if DEBUG:
print >> sys.stderr, 'tlm: Reading Session state from', config[
'state_dir']
nocachedb = cachedb.init(config, self.rawserver_fatalerrorfunc)
try_register(nocachedb, self.database_thread)
self.cat = Category.getInstance(config['install_dir'])
self.term = TermExtraction.getInstance(config['install_dir'])
self.peer_db = PeerDBHandler.getInstance()
self.peer_db.registerConnectionUpdater(self.session)
self.torrent_db = TorrentDBHandler.getInstance()
self.torrent_db.register(
os.path.abspath(config['torrent_collecting_dir']))
self.mypref_db = MyPreferenceDBHandler.getInstance()
self.votecast_db = VoteCastDBHandler.getInstance()
self.votecast_db.registerSession(self.session)
self.channelcast_db = ChannelCastDBHandler.getInstance()
self.channelcast_db.registerSession(self.session)
self.nb_db = NetworkBuzzDBHandler.getInstance()
self.ue_db = UserEventLogDBHandler.getInstance()
if self.dispersy:
self.dispersy.database.attach_commit_callback(
self.channelcast_db._db.commitNow)
else:
config['torrent_checking'] = 0
self.rtorrent_handler = None
if config['torrent_collecting']:
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
self.rtorrent_handler = RemoteTorrentHandler()
