def _task_check_reachability(self):
if self.get_session() and self.get_session().status(
).has_incoming_connections:
notify_reachability = lambda: self.notifier.notify(
NTFY_REACHABLE, NTFY_INSERT, None, '')
self.register_task(u'notify_reachability',
reactor.callLater(3, notify_reachability))
else:
self.register_task(
u'check_reachability',
reactor.callLater(10, self._task_check_reachability))
def test_can_create_engine_wrapper_retry(self):
impl = LibtorrentDownloadImpl(self.session, None)
impl.cew_scheduled = True
def set_cew_false():
self.session.lm.ltmgr.is_dht_ready = lambda: True
impl.cew_scheduled = False
# Simulate Tribler changing the cew, the calllater should have fired by now
# and before it executed the cew is false, firing the deferred.
reactor.callLater(2, set_cew_false)
return impl.can_create_engine_wrapper()
def initialize(self):
# start upnp
self.get_session().start_upnp()
# make temporary directory for metadata collecting through DHT
self.metadata_tmpdir = tempfile.mkdtemp(suffix=u'tribler_metainfo_tmpdir')
# register tasks
self.register_task(u'process_alerts', reactor.callLater(1, self._task_process_alerts))
self.register_task(u'check_reachability', reactor.callLater(1, self._task_check_reachability))
self._schedule_next_check(5, DHT_CHECK_RETRIES)
self.register_task(u'task_cleanup_metacache',
LoopingCall(self._task_cleanup_metainfo_cache)).start(60, now=True)
def schedule_download():
self.register_task(
"reschedule_dowload",
reactor.callLater(
5, reactor.callInThread, self.utility.
session.start_download_from_tdef, tdef,
dscfg))
def create_channel(self, name, description, mode, rss_url=None):
"""
Creates a new Channel.
:param name: Name of the Channel.
:param description: Description of the Channel.
:param mode: Mode of the Channel ('open', 'semi-open', or 'closed').
:param rss_url: RSS URL for the Channel.
"""
assert isinstance(
name, basestring), u"name is not a basestring: %s" % type(name)
assert isinstance(
description, basestring
), u"description is not a basestring: %s" % type(description)
assert mode in self._channel_mode_map, u"invalid mode: %s" % mode
assert isinstance(
rss_url,
basestring), u"rss_url is not a basestring: %s" % type(rss_url)
# if two channels have the same name, this will not work
if name in self._channel_callback_dict:
self._logger.error(u"Channel name already exists: %s", name)
return
def _create_channel():
channel_mode = self._channel_mode_map[mode]
community = ChannelCommunity.create_community(
self.dispersy,
self.session.dispersy_member,
tribler_session=self.session)
community.set_channel_mode(channel_mode)
community.create_channel(name, description)
self._channel_callback_dict[name] = {u'rss_url': rss_url}
self.register_task(u"create_channel_%s" % name,
reactor.callLater(0, _create_channel))
def create_channel(self, name, description, mode, rss_url=None):
"""
Creates a new Channel.
:param name: Name of the Channel.
:param description: Description of the Channel.
:param mode: Mode of the Channel ('open', 'semi-open', or 'closed').
:param rss_url: RSS URL for the Channel.
"""
assert isinstance(name, basestring), u"name is not a basestring: %s" % type(name)
assert isinstance(description, basestring), u"description is not a basestring: %s" % type(description)
assert mode in self._channel_mode_map, u"invalid mode: %s" % mode
assert isinstance(rss_url, basestring), u"rss_url is not a basestring: %s" % type(rss_url)
# if two channels have the same name, this will not work
if name in self._channel_callback_dict:
self._logger.error(u"Channel name already exists: %s", name)
return
def _create_channel():
channel_mode = self._channel_mode_map[mode]
community = ChannelCommunity.create_community(self.dispersy, self.session.dispersy_member,
tribler_session=self.session)
community.set_channel_mode(channel_mode)
community.create_channel(name, description)
self._channel_callback_dict[name] = {u'rss_url': rss_url}
self.register_task(u"create_channel_%s" % name, reactor.callLater(0, _create_channel))
def dispersy_started(self, *args):
default_kwargs = {'tribler_session': self.session}
dispersy = self.session.get_dispersy_instance()
dispersy.define_auto_load(ChannelCommunity, self.session.dispersy_member, load=True, kargs=default_kwargs)
dispersy.define_auto_load(PreviewChannelCommunity, self.session.dispersy_member, kargs=default_kwargs)
self.register_task(u'prepare_channels', reactor.callLater(10, self._prepare_channels))
def _task_process_alerts(self):
for ltsession in self.ltsessions.itervalues():
if ltsession:
alert = ltsession.pop_alert()
while alert:
self.process_alert(alert)
alert = ltsession.pop_alert()
self.register_task(u'process_alerts', reactor.callLater(1, self._task_process_alerts))
def initialize(self):
# start upnp
self.get_session().start_upnp()
# make temporary directory for metadata collecting through DHT
self.metadata_tmpdir = tempfile.mkdtemp(
suffix=u'tribler_metainfo_tmpdir')
# register tasks
self.register_task(u'process_alerts',
reactor.callLater(1, self._task_process_alerts))
self.register_task(u'check_reachability',
reactor.callLater(1, self._task_check_reachability))
self._schedule_next_check(5, DHT_CHECK_RETRIES)
self.register_task(
u'task_cleanup_metacache',
LoopingCall(self._task_cleanup_metainfo_cache)).start(60, now=True)
def _task_process_alerts(self):
for ltsession in self.ltsessions.itervalues():
if ltsession:
alert = ltsession.pop_alert()
while alert:
self.process_alert(alert)
alert = ltsession.pop_alert()
self.register_task(u'process_alerts',
reactor.callLater(1, self._task_process_alerts))
def check_torrents_channel(src, defer_param=None):
"""
check if a torrent already in channel and ready to download
"""
if defer_param is None:
defer_param = defer.Deferred()
src_obj = self.boosting_manager.get_source_object(src)
success = True
if len(src_obj.unavail_torrent) == 0:
self.assertLessEqual(len(src_obj.torrents),
src_obj.max_torrents)
else:
success = False
reactor.callLater(1, check_torrents_channel, src, defer_param)
if success:
src_obj.community.cancel_all_pending_tasks()
src_obj.kill_tasks()
defer_param.callback(src)
return defer_param
def initialize(self):
# initialize URL cache
url_cache_name = u"rss_cache_%s.txt" % binascii.hexlify(self.channel_community.cid)
url_cache_path = os.path.join(self.session.get_state_dir(), url_cache_name)
self._url_cache = SimpleCache(url_cache_path)
self._url_cache.load()
# create temporary directory
self._tmp_dir = tempfile.mkdtemp()
# schedule the scraping task
self.register_task(u"rss_scrape",
reactor.callLater(2, self._task_scrape))
def _task_scrape(self):
rss_parser = RSSFeedParser()
for rss_item in rss_parser.parse(self.rss_url, self._url_cache):
if self._to_stop:
return
torrent_deferred = getPage(rss_item[u'torrent_url'].encode('utf-8'))
torrent_deferred.addCallback(lambda t, r=rss_item: self.on_got_torrent(t, rss_item=r))
if not self._to_stop:
# schedule the next scraping task
self._logger.info(u"Finish scraping %s, schedule task after %s", self.rss_url, self.check_interval)
self.register_task(u'rss_scrape',
reactor.callLater(self.check_interval, self._task_scrape))
def setUp(self):
self.assertFalse(Session.has_instance(), 'A session instance is already present when setting up the test')
AbstractServer.setUp(self, annotate=False)
self.app = wx.GetApp()
if not self.app:
from Tribler.Main.tribler_main import TriblerApp
self.app = TriblerApp(redirect=False)
self.guiUtility = None
self.frame = None
self.lm = None
self.session = None
self.hadSession = False
self.quitting = False
self.asserts = []
self.annotate(self._testMethodName, start=True)
self.wx_watchdog = Event()
self.twisted_watchdog = Event()
def wx_watchdog_keepalive():
if self.wx_watchdog:
self.wx_watchdog.set()
wx.CallLater(500, wx_watchdog_keepalive)
wx_watchdog_keepalive()
def twisted_watchdog_keepalive():
if self.twisted_watchdog:
self.twisted_watchdog.set()
reactor.callLater(0.5, twisted_watchdog_keepalive)
reactor.callLater(0.5, twisted_watchdog_keepalive)
self.watchdog.register_event(self.wx_watchdog, "wx thread")
self.watchdog.register_event(self.twisted_watchdog, "twisted thread")
def initialize(self):
# initialize URL cache
url_cache_name = u"rss_cache_%s.txt" % binascii.hexlify(
self.channel_community.cid)
url_cache_path = os.path.join(self.session.get_state_dir(),
url_cache_name)
self._url_cache = SimpleCache(url_cache_path)
self._url_cache.load()
# create temporary directory
self._tmp_dir = tempfile.mkdtemp()
# schedule the scraping task
self.register_task(u"rss_scrape",
reactor.callLater(2, self._task_scrape))
def _task_scrape(self):
rss_parser = RSSFeedParser()
for rss_item in rss_parser.parse(self.rss_url, self._url_cache):
if self._to_stop:
return
torrent_deferred = getPage(
rss_item[u'torrent_url'].encode('utf-8'))
torrent_deferred.addCallback(
lambda t, r=rss_item: self.on_got_torrent(t, rss_item=r))
if not self._to_stop:
# schedule the next scraping task
self._logger.info(u"Finish scraping %s, schedule task after %s",
self.rss_url, self.check_interval)
self.register_task(
u'rss_scrape',
reactor.callLater(self.check_interval, self._task_scrape))
def check_torrents_channel(src, defer_param=None, target=1):
"""
check if a torrent already in channel and ready to download
"""
if defer_param is None:
defer_param = defer.Deferred()
src_obj = self.boosting_manager.get_source_object(src)
success = True
if not src_obj or len(src_obj.torrents) < target:
success = False
reactor.callLater(1,
check_torrents_channel,
src,
defer_param,
target=target)
elif not self.boosting_manager.torrents.get(
TORRENT_FILE_INFOHASH, None):
success = False
reactor.callLater(1,
check_torrents_channel,
src,
defer_param,
target=target)
elif not self.boosting_manager.torrents[TORRENT_FILE_INFOHASH].get(
'download', None):
success = False
reactor.callLater(1,
check_torrents_channel,
src,
defer_param,
target=target)
if success:
self.boosting_manager.set_enable_mining(src,
False,
force_restart=True)
if src_obj.community:
src_obj.community.cancel_all_pending_tasks()
defer_param.callback(src)
return defer_param
def _schedule_next_check(self, delay, retries_left):
self.register_task(
u'check_dht',
reactor.callLater(delay, self._task_check_dht, retries_left))
def _task_check_reachability(self):
if self.get_session() and self.get_session().status().has_incoming_connections:
notify_reachability = lambda: self.notifier.notify(NTFY_REACHABLE, NTFY_INSERT, None, '')
self.register_task(u'notify_reachability', reactor.callLater(3, notify_reachability))
else:
self.register_task(u'check_reachability', reactor.callLater(10, self._task_check_reachability))
def _schedule_next_check(self, delay, retries_left):
self.register_task(u'check_dht', reactor.callLater(delay, self._task_check_dht, retries_left))
def test_chn_max_torrents(self):
"""
Test the restriction of max_torrents in a source.
"""
self.session.get_dispersy = lambda: True
self.session.lm.dispersy = Dispersy(ManualEnpoint(0),
self.getStateDir())
dispersy_cid_hex = "abcd" * 9 + "0012"
dispersy_cid = binascii.unhexlify(dispersy_cid_hex)
# create channel and insert torrent
self.create_fake_allchannel_community()
self.create_torrents_in_channel(dispersy_cid_hex)
pioneer_file = os.path.join(
TESTS_DATA_DIR, "Pioneer.One.S01E06.720p.x264-VODO.torrent")
pioneer_tdef = TorrentDef.load(pioneer_file)
pioneer_ihash = binascii.unhexlify(
"66ED7F30E3B30FA647ABAA19A36E7503AA071535")
torrent_list = [[
self.channel_id, 1, 1, pioneer_ihash, 1460000001, pioneer_file,
pioneer_tdef.get_files_as_unicode_with_length(),
pioneer_tdef.get_trackers_as_single_tuple()
]]
self.insert_torrents_into_channel(torrent_list)
self.boosting_manager.add_source(dispersy_cid)
chn_obj = self.boosting_manager.get_source_object(dispersy_cid)
chn_obj.max_torrents = 2
chn_obj.torrent_mgr.load_torrent = lambda dummy_1, dummy_2: None
def _load(torrent, callback=None):
if not isinstance(torrent, CollectedTorrent):
torrent_id = 0
if torrent.torrent_id <= 0:
torrent_id = self.session.lm.torrent_db.getTorrentID(
torrent.infohash)
if torrent_id:
torrent.update_torrent_id(torrent_id)
infohash_str = binascii.hexlify(torrent.infohash)
torrent = CollectedTorrent(
torrent, self.tdef
if infohash_str.startswith("fc") else pioneer_tdef)
if callback is not None:
callback(torrent)
else:
return torrent
def activate_mgr():
"""
activate ltmgr and adjust max torrents to emulate overflow torrents
"""
chn_obj.max_torrents = 1
chn_obj.torrent_mgr.load_torrent = _load
reactor.callLater(5, activate_mgr)
def check_torrents_channel(src, defer_param=None):
"""
check if a torrent already in channel and ready to download
"""
if defer_param is None:
defer_param = defer.Deferred()
src_obj = self.boosting_manager.get_source_object(src)
success = True
if len(src_obj.unavail_torrent) == 0:
self.assertLessEqual(len(src_obj.torrents),
src_obj.max_torrents)
else:
success = False
reactor.callLater(1, check_torrents_channel, src, defer_param)
if success:
src_obj.community.cancel_all_pending_tasks()
src_obj.kill_tasks()
defer_param.callback(src)
return defer_param
d = self.check_source(dispersy_cid)
d.addCallback(check_torrents_channel)
return d
def test_chn_exist_lookup(self):
"""
testing existing channel as a source.
It also tests how boosting manager cope with unknown channel with retrying
the lookup
"""
self.session.get_dispersy = lambda: True
self.session.lm.dispersy = Dispersy(ManualEnpoint(0),
self.getStateDir())
dispersy_cid_hex = "abcd" * 9 + "0012"
dispersy_cid = binascii.unhexlify(dispersy_cid_hex)
# create channel and insert torrent
self.create_fake_allchannel_community()
self.create_torrents_in_channel(dispersy_cid_hex)
# channel is exist
community = ChannelCommunity.init_community(
self.session.lm.dispersy,
self.session.lm.dispersy.get_member(mid=dispersy_cid),
self.session.lm.dispersy._communities['allchannel']._my_member,
self.session)
# make the id unknown so boosting manager can test repeating search
id_tmp = community._channel_id
community._channel_id = 0
def _set_id_channel(channel_id):
"""
set channel id manually (emulate finding)
"""
community._channel_id = channel_id
reactor.callLater(5, _set_id_channel, id_tmp)
self.boosting_manager.add_source(dispersy_cid)
chn_obj = self.boosting_manager.get_source_object(dispersy_cid)
def _load(torrent, callback=None):
if not isinstance(torrent, CollectedTorrent):
torrent_id = 0
if torrent.torrent_id <= 0:
torrent_id = self.session.lm.torrent_db.getTorrentID(
torrent.infohash)
if torrent_id:
torrent.update_torrent_id(torrent_id)
torrent = CollectedTorrent(torrent, self.tdef)
if callback is not None:
callback(torrent)
else:
return torrent
chn_obj.torrent_mgr.load_torrent = _load
def clean_community(_):
"""
cleanly exit the community we are in
"""
if chn_obj.community:
chn_obj.community.cancel_all_pending_tasks()
chn_obj.kill_tasks()
d = self.check_source(dispersy_cid)
d.addCallback(clean_community)
return d
def twisted_watchdog_keepalive():
if self.twisted_watchdog:
self.twisted_watchdog.set()
reactor.callLater(0.5, twisted_watchdog_keepalive)
