def setUp(self, autoload_discovery=True):
yield super(TestBoostingManagerSys, self).setUp()
self.set_boosting_settings()
self.session.lm.ltmgr.get_session(
).find_torrent = lambda _: MockLtTorrent()
self.boosting_manager = BoostingManager(self.session, self.bsettings)
self.session.lm.boosting_manager = self.boosting_manager
class TriblerLaunchMany(TaskManager):
def __init__(self):
""" Called only once (unless we have multiple Sessions) by MainThread """
super(TriblerLaunchMany, self).__init__()
self.initComplete = False
self.registered = False
self.dispersy = None
self.state_cb_count = 0
self.previous_active_downloads = []
self.download_states_lc = None
self.get_peer_list = []
self._logger = logging.getLogger(self.__class__.__name__)
self.downloads = {}
self.upnp_ports = []
self.session = None
self.session_lock = None
self.sessdoneflag = Event()
self.shutdownstarttime = None
# modules
self.torrent_store = None
self.metadata_store = None
self.rtorrent_handler = None
self.tftp_handler = None
self.api_manager = None
self.watch_folder = None
self.version_check_manager = None
self.resource_monitor = None
self.category = None
self.peer_db = None
self.torrent_db = None
self.mypref_db = None
self.votecast_db = None
self.channelcast_db = None
self.search_manager = None
self.channel_manager = None
self.video_server = None
self.mainline_dht = None
self.ltmgr = None
self.tracker_manager = None
self.torrent_checker = None
self.tunnel_community = None
self.startup_deferred = Deferred()
self.boosting_manager = None
self.market_community = None
def register(self, session, session_lock):
assert isInIOThread()
if not self.registered:
self.registered = True
self.session = session
self.session_lock = session_lock
# On Mac, we bundle the root certificate for the SSL validation since Twisted is not using the root
# certificates provided by the system trust store.
if sys.platform == 'darwin':
os.environ['SSL_CERT_FILE'] = os.path.join(
get_lib_path(), 'root_certs_mac.pem')
if self.session.config.get_torrent_store_enabled():
from Tribler.Core.leveldbstore import LevelDbStore
self.torrent_store = LevelDbStore(
self.session.config.get_torrent_store_dir())
if self.session.config.get_metadata_enabled():
from Tribler.Core.leveldbstore import LevelDbStore
self.metadata_store = LevelDbStore(
self.session.config.get_metadata_store_dir())
# torrent collecting: RemoteTorrentHandler
if self.session.config.get_torrent_collecting_enabled():
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
self.rtorrent_handler = RemoteTorrentHandler(self.session)
# TODO(emilon): move this to a megacache component or smth
if self.session.config.get_megacache_enabled():
from Tribler.Core.CacheDB.SqliteCacheDBHandler import (
PeerDBHandler, TorrentDBHandler, MyPreferenceDBHandler,
VoteCastDBHandler, ChannelCastDBHandler)
from Tribler.Core.Category.Category import Category
self._logger.debug('tlm: Reading Session state from %s',
self.session.config.get_state_dir())
self.category = Category()
# create DBHandlers
self.peer_db = PeerDBHandler(self.session)
self.torrent_db = TorrentDBHandler(self.session)
self.mypref_db = MyPreferenceDBHandler(self.session)
self.votecast_db = VoteCastDBHandler(self.session)
self.channelcast_db = ChannelCastDBHandler(self.session)
# initializes DBHandlers
self.peer_db.initialize()
self.torrent_db.initialize()
self.mypref_db.initialize()
self.votecast_db.initialize()
self.channelcast_db.initialize()
from Tribler.Core.Modules.tracker_manager import TrackerManager
self.tracker_manager = TrackerManager(self.session)
if self.session.config.get_video_server_enabled():
self.video_server = VideoServer(
self.session.config.get_video_server_port(), self.session)
self.video_server.start()
# Dispersy
self.tftp_handler = None
if self.session.config.get_dispersy_enabled():
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import StandaloneEndpoint
# set communication endpoint
endpoint = StandaloneEndpoint(
self.session.config.get_dispersy_port())
working_directory = unicode(
self.session.config.get_state_dir())
self.dispersy = Dispersy(endpoint, working_directory)
# register TFTP service
from Tribler.Core.TFTP.handler import TftpHandler
self.tftp_handler = TftpHandler(self.session,
endpoint,
"fffffffd".decode('hex'),
block_size=1024)
self.tftp_handler.initialize()
if self.session.config.get_torrent_search_enabled(
) or self.session.config.get_channel_search_enabled():
self.search_manager = SearchManager(self.session)
self.search_manager.initialize()
if not self.initComplete:
self.init()
self.session.add_observer(self.on_tribler_started, NTFY_TRIBLER,
[NTFY_STARTED])
self.session.notifier.notify(NTFY_TRIBLER, NTFY_STARTED, None)
return self.startup_deferred
def on_tribler_started(self, subject, changetype, objectID, *args):
reactor.callFromThread(self.startup_deferred.callback, None)
@blocking_call_on_reactor_thread
def load_communities(self):
self._logger.info("tribler: Preparing communities...")
now_time = timemod.time()
default_kwargs = {'tribler_session': self.session}
# Search Community
if self.session.config.get_torrent_search_enabled():
from Tribler.community.search.community import SearchCommunity
self.dispersy.define_auto_load(SearchCommunity,
self.session.dispersy_member,
load=True,
kargs=default_kwargs)
# AllChannel Community
if self.session.config.get_channel_search_enabled():
from Tribler.community.allchannel.community import AllChannelCommunity
self.dispersy.define_auto_load(AllChannelCommunity,
self.session.dispersy_member,
load=True,
kargs=default_kwargs)
# Channel Community
if self.session.config.get_channel_community_enabled():
from Tribler.community.channel.community import ChannelCommunity
self.dispersy.define_auto_load(ChannelCommunity,
self.session.dispersy_member,
load=True,
kargs=default_kwargs)
# PreviewChannel Community
if self.session.config.get_preview_channel_community_enabled():
from Tribler.community.channel.preview import PreviewChannelCommunity
self.dispersy.define_auto_load(PreviewChannelCommunity,
self.session.dispersy_member,
kargs=default_kwargs)
# Tunnel Community
mc_community = None
if self.session.config.get_tunnel_community_enabled():
tunnel_settings = TunnelSettings(tribler_session=self.session)
tunnel_kwargs = {
'tribler_session': self.session,
'settings': tunnel_settings
}
if self.session.config.get_trustchain_enabled():
trustchain_kwargs = {'tribler_session': self.session}
# If the trustchain is enabled, we use the permanent trustchain keypair
# for both the trustchain and the tunnel community
keypair = self.session.trustchain_keypair
dispersy_member = self.dispersy.get_member(
private_key=keypair.key_to_bin())
from Tribler.community.triblerchain.community import TriblerChainCommunity
mc_community = self.dispersy.define_auto_load(
TriblerChainCommunity,
dispersy_member,
load=True,
kargs=trustchain_kwargs)[0]
else:
keypair = self.dispersy.crypto.generate_key(u"curve25519")
dispersy_member = self.dispersy.get_member(
private_key=self.dispersy.crypto.key_to_bin(keypair))
from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity
self.tunnel_community = self.dispersy.define_auto_load(
HiddenTunnelCommunity,
dispersy_member,
load=True,
kargs=tunnel_kwargs)[0]
# We don't want to automatically load other instances of this community with other master members.
self.dispersy.undefine_auto_load(HiddenTunnelCommunity)
# Use the permanent TrustChain ID for Market community/TradeChain if it's available
if self.session.config.get_market_community_enabled():
wallets = {}
btc_wallet = BitcoinWallet(
os.path.join(self.session.config.get_state_dir(), 'wallet'),
testnet=self.session.config.get_btc_testnet())
wallets[btc_wallet.get_identifier()] = btc_wallet
mc_wallet = TrustchainWallet(mc_community)
wallets[mc_wallet.get_identifier()] = mc_wallet
if self.session.config.get_dummy_wallets_enabled():
# For debugging purposes, we create dummy wallets
dummy_wallet1 = DummyWallet1()
wallets[dummy_wallet1.get_identifier()] = dummy_wallet1
dummy_wallet2 = DummyWallet2()
wallets[dummy_wallet2.get_identifier()] = dummy_wallet2
from Tribler.community.market.community import MarketCommunity
keypair = self.session.tradechain_keypair
dispersy_member = self.dispersy.get_member(
private_key=keypair.key_to_bin())
market_kwargs = {
'tribler_session': self.session,
'wallets': wallets
}
self.market_community = self.dispersy.define_auto_load(
MarketCommunity,
dispersy_member,
load=True,
kargs=market_kwargs)[0]
self.session.config.set_anon_proxy_settings(
2,
("127.0.0.1",
self.session.config.get_tunnel_community_socks5_listen_ports()))
self._logger.info("tribler: communities are ready in %.2f seconds",
timemod.time() - now_time)
def init(self):
if self.dispersy:
from Tribler.dispersy.community import HardKilledCommunity
self._logger.info("lmc: Starting Dispersy...")
now = timemod.time()
success = self.dispersy.start(self.session.autoload_discovery)
diff = timemod.time() - now
if success:
self._logger.info(
"lmc: Dispersy started successfully in %.2f seconds [port: %d]",
diff, self.dispersy.wan_address[1])
else:
self._logger.info(
"lmc: Dispersy failed to start in %.2f seconds", diff)
self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))
from Tribler.dispersy.crypto import M2CryptoSK
private_key = self.dispersy.crypto.key_to_bin(
M2CryptoSK(filename=self.session.config.
get_permid_keypair_filename()))
self.session.dispersy_member = blockingCallFromThread(
reactor, self.dispersy.get_member, private_key=private_key)
blockingCallFromThread(reactor,
self.dispersy.define_auto_load,
HardKilledCommunity,
self.session.dispersy_member,
load=True)
if self.session.config.get_megacache_enabled():
self.dispersy.database.attach_commit_callback(
self.session.sqlite_db.commit_now)
# notify dispersy finished loading
self.session.notifier.notify(NTFY_DISPERSY, NTFY_STARTED, None)
self.load_communities()
tunnel_community_ports = self.session.config.get_tunnel_community_socks5_listen_ports(
)
self.session.config.set_anon_proxy_settings(
2, ("127.0.0.1", tunnel_community_ports))
if self.session.config.get_channel_search_enabled():
from Tribler.Core.Modules.channel.channel_manager import ChannelManager
self.channel_manager = ChannelManager(self.session)
self.channel_manager.initialize()
if self.session.config.get_mainline_dht_enabled():
from Tribler.Core.DecentralizedTracking import mainlineDHT
self.mainline_dht = mainlineDHT.init(
('127.0.0.1', self.session.config.get_mainline_dht_port()),
self.session.config.get_state_dir())
self.upnp_ports.append(
(self.session.config.get_mainline_dht_port(), 'UDP'))
if self.session.config.get_libtorrent_enabled():
from Tribler.Core.Libtorrent.LibtorrentMgr import LibtorrentMgr
self.ltmgr = LibtorrentMgr(self.session)
self.ltmgr.initialize()
for port, protocol in self.upnp_ports:
self.ltmgr.add_upnp_mapping(port, protocol)
# add task for tracker checking
if self.session.config.get_torrent_checking_enabled():
self.torrent_checker = TorrentChecker(self.session)
self.torrent_checker.initialize()
if self.rtorrent_handler:
self.rtorrent_handler.initialize()
if self.api_manager:
self.api_manager.root_endpoint.start_endpoints()
if self.session.config.get_watch_folder_enabled():
self.watch_folder = WatchFolder(self.session)
self.watch_folder.start()
if self.session.config.get_credit_mining_enabled():
from Tribler.Core.CreditMining.BoostingManager import BoostingManager
self.boosting_manager = BoostingManager(self.session)
if self.session.config.get_resource_monitor_enabled():
self.resource_monitor = ResourceMonitor(self.session)
self.resource_monitor.start()
self.version_check_manager = VersionCheckManager(self.session)
self.session.set_download_states_callback(self.sesscb_states_callback)
self.initComplete = True
def add(self,
tdef,
dscfg,
pstate=None,
setupDelay=0,
hidden=False,
share_mode=False,
checkpoint_disabled=False):
""" Called by any thread """
d = None
with self.session_lock:
if not isinstance(
tdef, TorrentDefNoMetainfo) and not tdef.is_finalized():
raise ValueError("TorrentDef not finalized")
infohash = tdef.get_infohash()
# Create the destination directory if it does not exist yet
try:
if not os.path.isdir(dscfg.get_dest_dir()):
os.makedirs(dscfg.get_dest_dir())
except OSError:
self._logger.error(
"Unable to create the download destination directory.")
if dscfg.get_time_added() == 0:
dscfg.set_time_added(int(timemod.time()))
# Check if running or saved on disk
if infohash in self.downloads:
raise DuplicateDownloadException(
"This download already exists.")
from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl
d = LibtorrentDownloadImpl(self.session, tdef)
if pstate is None: # not already resuming
pstate = self.load_download_pstate_noexc(infohash)
if pstate is not None:
self._logger.debug("tlm: add: pstate is %s %s",
pstate.get('dlstate', 'status'),
pstate.get('dlstate', 'progress'))
# Store in list of Downloads, always.
self.downloads[infohash] = d
setup_deferred = d.setup(dscfg,
pstate,
wrapperDelay=setupDelay,
share_mode=share_mode,
checkpoint_disabled=checkpoint_disabled)
setup_deferred.addCallback(self.on_download_handle_created)
if d and not hidden and self.session.config.get_megacache_enabled():
@forceDBThread
def write_my_pref():
torrent_id = self.torrent_db.getTorrentID(infohash)
data = {'destination_path': d.get_dest_dir()}
self.mypref_db.addMyPreference(torrent_id, data)
if isinstance(tdef, TorrentDefNoMetainfo):
self.torrent_db.addOrGetTorrentID(tdef.get_infohash())
self.torrent_db.updateTorrent(tdef.get_infohash(),
name=tdef.get_name_as_unicode())
write_my_pref()
elif self.rtorrent_handler:
self.rtorrent_handler.save_torrent(tdef, write_my_pref)
else:
self.torrent_db.addExternalTorrent(
tdef, extra_info={'status': 'good'})
write_my_pref()
return d
def on_download_handle_created(self, download):
"""
This method is called when the download handle has been created.
Immediately checkpoint the download and write the resume data.
"""
return download.checkpoint()
def remove(self, d, removecontent=False, removestate=True, hidden=False):
""" Called by any thread """
out = None
with self.session_lock:
out = d.stop_remove(removestate=removestate,
removecontent=removecontent)
infohash = d.get_def().get_infohash()
if infohash in self.downloads:
del self.downloads[infohash]
if not hidden:
self.remove_id(infohash)
if self.tunnel_community:
self.tunnel_community.on_download_removed(d)
return out or succeed(None)
def remove_id(self, infohash):
@forceDBThread
def do_db():
torrent_id = self.torrent_db.getTorrentID(infohash)
if torrent_id:
self.mypref_db.deletePreference(torrent_id)
if self.session.config.get_megacache_enabled():
do_db()
def get_downloads(self):
""" Called by any thread """
with self.session_lock:
return self.downloads.values() # copy, is mutable
def get_download(self, infohash):
""" Called by any thread """
with self.session_lock:
return self.downloads.get(infohash, None)
def download_exists(self, infohash):
with self.session_lock:
return infohash in self.downloads
@blocking_call_on_reactor_thread
@inlineCallbacks
def update_download_hops(self, download, new_hops):
"""
Update the amount of hops for a specified download. This can be done on runtime.
"""
infohash = binascii.hexlify(download.tdef.get_infohash())
self._logger.info("Updating the amount of hops of download %s",
infohash)
yield self.session.remove_download(download)
# copy the old download_config and change the hop count
dscfg = download.copy()
dscfg.set_hops(new_hops)
self.session.start_download_from_tdef(download.tdef, dscfg)
def update_trackers(self, infohash, trackers):
""" Update the trackers for a download.
:param infohash: infohash of the torrent that needs to be updated
:param trackers: A list of tracker urls.
"""
dl = self.get_download(infohash)
old_def = dl.get_def() if dl else None
if old_def:
old_trackers = old_def.get_trackers_as_single_tuple()
new_trackers = list(set(trackers) - set(old_trackers))
all_trackers = list(old_trackers) + new_trackers
if new_trackers:
# Add new trackers to the download
dl.add_trackers(new_trackers)
# Create a new TorrentDef
if isinstance(old_def, TorrentDefNoMetainfo):
new_def = TorrentDefNoMetainfo(old_def.get_infohash(),
old_def.get_name(),
dl.get_magnet_link())
else:
metainfo = old_def.get_metainfo()
if len(all_trackers) > 1:
metainfo["announce-list"] = [all_trackers]
else:
metainfo["announce"] = all_trackers[0]
new_def = TorrentDef.load_from_dict(metainfo)
# Set TorrentDef + checkpoint
dl.set_def(new_def)
dl.checkpoint()
if isinstance(old_def, TorrentDefNoMetainfo):
@forceDBThread
def update_trackers_db(infohash, new_trackers):
torrent_id = self.torrent_db.getTorrentID(infohash)
if torrent_id is not None:
self.torrent_db.addTorrentTrackerMappingInBatch(
torrent_id, new_trackers)
self.session.notifier.notify(
NTFY_TORRENTS, NTFY_UPDATE, infohash)
if self.session.config.get_megacache_enabled():
update_trackers_db(infohash, new_trackers)
elif not isinstance(
old_def,
TorrentDefNoMetainfo) and self.rtorrent_handler:
# Update collected torrents
self.rtorrent_handler.save_torrent(new_def)
#
# State retrieval
#
def stop_download_states_callback(self):
"""
Stop any download states callback if present.
"""
if self.is_pending_task_active("download_states_lc"):
self.cancel_pending_task("download_states_lc")
def set_download_states_callback(self, user_callback, interval=1.0):
"""
Set the download state callback. Remove any old callback if it's present.
"""
self.stop_download_states_callback()
self._logger.debug(
"Starting the download state callback with interval %f", interval)
self.download_states_lc = self.register_task(
"download_states_lc",
LoopingCall(self._invoke_states_cb, user_callback))
self.download_states_lc.start(interval)
def _invoke_states_cb(self, callback):
"""
Invoke the download states callback with a list of the download states.
"""
dslist = []
for d in self.downloads.values():
d.set_moreinfo_stats(
True in self.get_peer_list
or d.get_def().get_infohash() in self.get_peer_list)
ds = d.network_get_state(None, False)
dslist.append(ds)
def on_cb_done(new_get_peer_list):
self.get_peer_list = new_get_peer_list
return deferToThread(callback, dslist).addCallback(on_cb_done)
def sesscb_states_callback(self, states_list):
"""
This method is periodically (every second) called with a list of the download states of the active downloads.
"""
self.state_cb_count += 1
# Check to see if a download has finished
new_active_downloads = []
do_checkpoint = False
seeding_download_list = []
for ds in states_list:
state = ds.get_status()
download = ds.get_download()
tdef = download.get_def()
safename = tdef.get_name_as_unicode()
if state == DLSTATUS_DOWNLOADING:
new_active_downloads.append(safename)
elif state == DLSTATUS_STOPPED_ON_ERROR:
self._logger.error("Error during download: %s",
repr(ds.get_error()))
self.downloads.get(tdef.get_infohash()).stop()
self.session.notifier.notify(NTFY_TORRENT, NTFY_ERROR,
tdef.get_infohash(),
repr(ds.get_error()))
elif state == DLSTATUS_SEEDING:
seeding_download_list.append({
u'infohash': tdef.get_infohash(),
u'download': download
})
if safename in self.previous_active_downloads:
self.session.notifier.notify(NTFY_TORRENT, NTFY_FINISHED,
tdef.get_infohash(), safename)
do_checkpoint = True
self.previous_active_downloads = new_active_downloads
if do_checkpoint:
self.session.checkpoint_downloads()
if self.state_cb_count % 4 == 0 and self.tunnel_community:
self.tunnel_community.monitor_downloads(states_list)
return []
#
# Persistence methods
#
def load_checkpoint(self):
""" Called by any thread """
def do_load_checkpoint():
with self.session_lock:
for i, filename in enumerate(
iglob(
os.path.join(
self.session.get_downloads_pstate_dir(),
'*.state'))):
self.resume_download(filename, setupDelay=i * 0.1)
if self.initComplete:
do_load_checkpoint()
else:
self.register_task("load_checkpoint",
reactor.callLater(1, do_load_checkpoint))
def load_download_pstate_noexc(self, infohash):
""" Called by any thread, assume session_lock already held """
try:
basename = binascii.hexlify(infohash) + '.state'
filename = os.path.join(self.session.get_downloads_pstate_dir(),
basename)
if os.path.exists(filename):
return self.load_download_pstate(filename)
else:
self._logger.info("%s not found", basename)
except Exception:
self._logger.exception("Exception while loading pstate: %s",
infohash)
def resume_download(self, filename, setupDelay=0):
tdef = dscfg = pstate = None
try:
pstate = self.load_download_pstate(filename)
# SWIFTPROC
metainfo = pstate.get('state', 'metainfo')
if 'infohash' in metainfo:
tdef = TorrentDefNoMetainfo(metainfo['infohash'],
metainfo['name'],
metainfo.get('url', None))
else:
tdef = TorrentDef.load_from_dict(metainfo)
if pstate.has_option('download_defaults', 'saveas') and \
isinstance(pstate.get('download_defaults', 'saveas'), tuple):
pstate.set('download_defaults', 'saveas',
pstate.get('download_defaults', 'saveas')[-1])
dscfg = DownloadStartupConfig(pstate)
except:
# pstate is invalid or non-existing
_, file = os.path.split(filename)
infohash = binascii.unhexlify(file[:-6])
torrent_data = self.torrent_store.get(infohash)
if torrent_data:
try:
tdef = TorrentDef.load_from_memory(torrent_data)
defaultDLConfig = DefaultDownloadStartupConfig.getInstance(
)
dscfg = defaultDLConfig.copy()
if self.mypref_db is not None:
dest_dir = self.mypref_db.getMyPrefStatsInfohash(
infohash)
if dest_dir and os.path.isdir(dest_dir):
dscfg.set_dest_dir(dest_dir)
except ValueError:
self._logger.warning("tlm: torrent data invalid")
if pstate is not None:
has_resume_data = pstate.get('state',
'engineresumedata') is not None
self._logger.debug(
"tlm: load_checkpoint: resumedata %s",
'len %s ' % len(pstate.get('state', 'engineresumedata'))
if has_resume_data else 'None')
if tdef and dscfg:
if dscfg.get_dest_dir() != '': # removed torrent ignoring
try:
if not self.download_exists(tdef.get_infohash()):
self.add(tdef, dscfg, pstate, setupDelay=setupDelay)
else:
self._logger.info(
"tlm: not resuming checkpoint because download has already been added"
)
except Exception as e:
self._logger.exception(
"tlm: load check_point: exception while adding download %s",
tdef)
else:
self._logger.info("tlm: removing checkpoint %s destdir is %s",
filename, dscfg.get_dest_dir())
os.remove(filename)
else:
self._logger.info("tlm: could not resume checkpoint %s %s %s",
filename, tdef, dscfg)
def checkpoint_downloads(self):
"""
Checkpoints all running downloads in Tribler.
Even if the list of Downloads changes in the mean time this is no problem.
For removals, dllist will still hold a pointer to the download, and additions are no problem
(just won't be included in list of states returned via callback).
"""
downloads = self.downloads.values()
deferred_list = []
self._logger.debug("tlm: checkpointing %s downloads", len(downloads))
for download in downloads:
deferred_list.append(download.checkpoint())
return DeferredList(deferred_list)
def shutdown_downloads(self):
"""
Shutdown all downloads in Tribler.
"""
for download in self.downloads.values():
download.stop()
def remove_pstate(self, infohash):
def do_remove():
if not self.download_exists(infohash):
dlpstatedir = self.session.get_downloads_pstate_dir()
# Remove checkpoint
hexinfohash = binascii.hexlify(infohash)
try:
basename = hexinfohash + '.state'
filename = os.path.join(dlpstatedir, basename)
self._logger.debug(
"remove pstate: removing dlcheckpoint entry %s",
filename)
if os.access(filename, os.F_OK):
os.remove(filename)
except:
# Show must go on
self._logger.exception("Could not remove state")
else:
self._logger.warning(
"remove pstate: download is back, restarted? Canceling removal! %s",
repr(infohash))
reactor.callFromThread(do_remove)
@inlineCallbacks
def early_shutdown(self):
""" Called as soon as Session shutdown is initiated. Used to start
shutdown tasks that takes some time and that can run in parallel
to checkpointing, etc.
:returns a Deferred that will fire once all dependencies acknowledge they have shutdown.
"""
self._logger.info("tlm: early_shutdown")
self.cancel_all_pending_tasks()
# Note: session_lock not held
self.shutdownstarttime = timemod.time()
if self.boosting_manager:
yield self.boosting_manager.shutdown()
self.boosting_manager = None
if self.torrent_checker:
yield self.torrent_checker.shutdown()
self.torrent_checker = None
if self.channel_manager:
yield self.channel_manager.shutdown()
self.channel_manager = None
if self.search_manager:
yield self.search_manager.shutdown()
self.search_manager = None
if self.rtorrent_handler:
yield self.rtorrent_handler.shutdown()
self.rtorrent_handler = None
if self.video_server:
yield self.video_server.shutdown_server()
self.video_server = None
if self.version_check_manager:
self.version_check_manager.stop()
self.version_check_manager = None
if self.resource_monitor:
self.resource_monitor.stop()
self.resource_monitor = None
self.tracker_manager = None
if self.dispersy:
self._logger.info("lmc: Shutting down Dispersy...")
now = timemod.time()
try:
success = yield self.dispersy.stop()
except:
print_exc()
success = False
diff = timemod.time() - now
if success:
self._logger.info(
"lmc: Dispersy successfully shutdown in %.2f seconds",
diff)
else:
self._logger.info(
"lmc: Dispersy failed to shutdown in %.2f seconds", diff)
if self.metadata_store is not None:
yield self.metadata_store.close()
self.metadata_store = None
if self.tftp_handler is not None:
yield self.tftp_handler.shutdown()
self.tftp_handler = None
if self.channelcast_db is not None:
yield self.channelcast_db.close()
self.channelcast_db = None
if self.votecast_db is not None:
yield self.votecast_db.close()
self.votecast_db = None
if self.mypref_db is not None:
yield self.mypref_db.close()
self.mypref_db = None
if self.torrent_db is not None:
yield self.torrent_db.close()
self.torrent_db = None
if self.peer_db is not None:
yield self.peer_db.close()
self.peer_db = None
if self.mainline_dht is not None:
from Tribler.Core.DecentralizedTracking import mainlineDHT
yield mainlineDHT.deinit(self.mainline_dht)
self.mainline_dht = None
if self.torrent_store is not None:
yield self.torrent_store.close()
self.torrent_store = None
if self.api_manager is not None:
yield self.api_manager.stop()
self.api_manager = None
if self.watch_folder is not None:
yield self.watch_folder.stop()
self.watch_folder = None
def network_shutdown(self):
try:
self._logger.info("tlm: network_shutdown")
ts = enumerate_threads()
self._logger.info("tlm: Number of threads still running %d",
len(ts))
for t in ts:
self._logger.info(
"tlm: Thread still running=%s, daemon=%s, instance=%s",
t.getName(), t.isDaemon(), t)
except:
print_exc()
# Stop network thread
self.sessdoneflag.set()
# Shutdown libtorrent session after checkpoints have been made
if self.ltmgr is not None:
self.ltmgr.shutdown()
self.ltmgr = None
def save_download_pstate(self, infohash, pstate):
""" Called by network thread """
self.downloads[infohash].pstate_for_restart = pstate
self.register_task("save_pstate %f" % timemod.clock(),
self.downloads[infohash].save_resume_data())
def load_download_pstate(self, filename):
""" Called by any thread """
pstate = CallbackConfigParser()
pstate.read_file(filename)
return pstate
def init(self):
if self.dispersy:
from Tribler.dispersy.community import HardKilledCommunity
self._logger.info("lmc: Starting Dispersy...")
now = timemod.time()
success = self.dispersy.start(self.session.autoload_discovery)
diff = timemod.time() - now
if success:
self._logger.info(
"lmc: Dispersy started successfully in %.2f seconds [port: %d]",
diff, self.dispersy.wan_address[1])
else:
self._logger.info(
"lmc: Dispersy failed to start in %.2f seconds", diff)
self.upnp_ports.append((self.dispersy.wan_address[1], 'UDP'))
from Tribler.dispersy.crypto import M2CryptoSK
private_key = self.dispersy.crypto.key_to_bin(
M2CryptoSK(filename=self.session.config.
get_permid_keypair_filename()))
self.session.dispersy_member = blockingCallFromThread(
reactor, self.dispersy.get_member, private_key=private_key)
blockingCallFromThread(reactor,
self.dispersy.define_auto_load,
HardKilledCommunity,
self.session.dispersy_member,
load=True)
if self.session.config.get_megacache_enabled():
self.dispersy.database.attach_commit_callback(
self.session.sqlite_db.commit_now)
# notify dispersy finished loading
self.session.notifier.notify(NTFY_DISPERSY, NTFY_STARTED, None)
self.load_communities()
tunnel_community_ports = self.session.config.get_tunnel_community_socks5_listen_ports(
)
self.session.config.set_anon_proxy_settings(
2, ("127.0.0.1", tunnel_community_ports))
if self.session.config.get_channel_search_enabled():
from Tribler.Core.Modules.channel.channel_manager import ChannelManager
self.channel_manager = ChannelManager(self.session)
self.channel_manager.initialize()
if self.session.config.get_mainline_dht_enabled():
from Tribler.Core.DecentralizedTracking import mainlineDHT
self.mainline_dht = mainlineDHT.init(
('127.0.0.1', self.session.config.get_mainline_dht_port()),
self.session.config.get_state_dir())
self.upnp_ports.append(
(self.session.config.get_mainline_dht_port(), 'UDP'))
if self.session.config.get_libtorrent_enabled():
from Tribler.Core.Libtorrent.LibtorrentMgr import LibtorrentMgr
self.ltmgr = LibtorrentMgr(self.session)
self.ltmgr.initialize()
for port, protocol in self.upnp_ports:
self.ltmgr.add_upnp_mapping(port, protocol)
# add task for tracker checking
if self.session.config.get_torrent_checking_enabled():
self.torrent_checker = TorrentChecker(self.session)
self.torrent_checker.initialize()
if self.rtorrent_handler:
self.rtorrent_handler.initialize()
if self.api_manager:
self.api_manager.root_endpoint.start_endpoints()
if self.session.config.get_watch_folder_enabled():
self.watch_folder = WatchFolder(self.session)
self.watch_folder.start()
if self.session.config.get_credit_mining_enabled():
from Tribler.Core.CreditMining.BoostingManager import BoostingManager
self.boosting_manager = BoostingManager(self.session)
if self.session.config.get_resource_monitor_enabled():
self.resource_monitor = ResourceMonitor(self.session)
self.resource_monitor.start()
self.version_check_manager = VersionCheckManager(self.session)
self.session.set_download_states_callback(self.sesscb_states_callback)
self.initComplete = True
class TestBoostingManagerSys(TestAsServer):
"""
Base class to test base credit mining function.
"""
def __init__(self, *argv, **kwargs):
super(TestBoostingManagerSys, self).__init__(*argv, **kwargs)
self._check_source_lc = None
self._check_torrents_lc = None
@blocking_call_on_reactor_thread
@inlineCallbacks
def setUp(self, autoload_discovery=True):
yield super(TestBoostingManagerSys, self).setUp()
self.set_boosting_settings()
self.session.lm.ltmgr.get_session(
).find_torrent = lambda _: MockLtTorrent()
self.boosting_manager = BoostingManager(self.session, self.bsettings)
self.session.lm.boosting_manager = self.boosting_manager
def set_boosting_settings(self):
"""
set settings in credit mining
"""
self.bsettings = BoostingSettings(self.session)
self.bsettings.credit_mining_path = os.path.join(
self.session_base_dir, "credit_mining")
self.bsettings.load_config = False
self.bsettings.check_dependencies = False
self.bsettings.min_connection_start = -1
self.bsettings.min_channels_start = -1
self.bsettings.max_torrents_active = 8
self.bsettings.max_torrents_per_source = 5
self.bsettings.tracker_interval = 5
self.bsettings.initial_tracker_interval = 5
self.bsettings.logging_interval = 30
self.bsettings.initial_logging_interval = 3
def setUpPreSession(self):
super(TestBoostingManagerSys, self).setUpPreSession()
self.config.set_torrent_checking(True)
self.config.set_megacache(True)
self.config.set_dispersy(True)
self.config.set_torrent_store(True)
self.config.set_enable_torrent_search(True)
self.config.set_enable_channel_search(True)
self.config.set_libtorrent(True)
@blocking_call_on_reactor_thread
@inlineCallbacks
def tearDown(self, annotate=True):
DefaultDownloadStartupConfig.delInstance()
self.boosting_manager.shutdown()
yield super(TestBoostingManagerSys, self).tearDown(annotate=annotate)
def check_torrents(self, src, target=1):
"""
Check if a specified number of torrent has been added to the passed source.
"""
defer_param = defer.Deferred()
def do_check():
src_obj = self.boosting_manager.get_source_object(src)
if src_obj and len(src_obj.torrents) >= target:
def _get_tor_dummy(_, keys=123, include_mypref=True):
"""
function to emulate get_torrent in torrent_db
"""
return {
'C.torrent_id': 93,
'category': u'Compressed',
'torrent_id': 41,
'infohash': src_obj.torrents.keys()[0],
'length': 1150844928,
'last_tracker_check': 10001,
'myDownloadHistory': False,
'name': u'ubuntu-15.04-desktop-amd64.iso',
'num_leechers': 999,
'num_seeders': 123,
'status': u'unknown',
'tracker_check_retries': 0
}
self.boosting_manager.torrent_db.getTorrent = _get_tor_dummy
self.session.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE,
src_obj.torrents.keys()[0])
# log it
self.boosting_manager.log_statistics()
self._check_torrents_lc.stop()
self._check_torrents_lc = None
defer_param.callback(src)
self._check_torrents_lc = LoopingCall(do_check)
self._check_torrents_lc.start(1, now=True)
return defer_param
def check_source(self, src):
"""
function to check if a source is ready initializing
"""
defer_param = defer.Deferred()
def do_check():
src_obj = self.boosting_manager.get_source_object(src)
if src_obj and src_obj.ready:
self._check_source_lc.stop()
self._check_source_lc = None
defer_param.callback(src)
self._check_source_lc = LoopingCall(do_check)
self._check_source_lc.start(1, now=True)
return defer_param