def create_fake_download_and_state():
"""
Create a fake download and state which can be passed to the global download callback.
"""
tdef = TorrentDef()
tdef.get_infohash = lambda: b'aaaa'
fake_peer = {'extended_version': 'Tribler', 'id': 'a' * 20, 'dtotal': 10 * 1024 * 1024}
fake_download = MockObject()
fake_download.get_def = lambda: tdef
fake_download.get_def().get_name_as_unicode = lambda: "test.iso"
fake_download.get_peerlist = lambda: [fake_peer]
fake_download.hidden = False
fake_download.checkpoint = lambda: succeed(None)
fake_download.stop = lambda: succeed(None)
fake_download.shutdown = lambda: succeed(None)
dl_state = MockObject()
dl_state.get_infohash = lambda: b'aaaa'
dl_state.get_status = lambda: DLSTATUS_SEEDING
dl_state.get_download = lambda: fake_download
fake_config = MockObject()
fake_config.get_hops = lambda: 0
fake_config.get_safe_seeding = lambda: True
fake_download.config = fake_config
return fake_download, dl_state
async def test_start_download(self):
"""
Testing the addition of a torrent to the libtorrent manager
"""
infohash = b'a' * 20
mock_handle = Mock()
mock_handle.info_hash = lambda: hexlify(infohash)
mock_handle.is_valid = lambda: True
mock_error = MockObject()
mock_error.value = lambda: None
mock_alert = type('add_torrent_alert', (object,), dict(handle=mock_handle,
error=mock_error,
category=lambda _: None))()
mock_ltsession = Mock()
mock_ltsession.get_torrents = lambda: []
mock_ltsession.async_add_torrent = lambda _: self.dlmgr.register_task('post_alert',
self.dlmgr.process_alert,
mock_alert, delay=0.1)
self.dlmgr.get_session = lambda *_: mock_ltsession
download = self.dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, ''), checkpoint_disabled=True)
handle = await download.get_handle()
self.assertEqual(handle, mock_handle)
# Test waiting on DHT getting enough nodes and adding the torrent after timing out
self.dlmgr.dht_readiness_timeout = 0.5
flag = []
check_was_run = Mock()
async def mock_check():
while not flag:
check_was_run()
await sleep(0.1)
self.dlmgr._check_dht_ready = mock_check
self.dlmgr.initialize()
mock_download = Mock()
mock_download.get_def().get_infohash = lambda: b"1"*20
mock_download.future_added = succeed(True)
mock_ltsession.async_add_torrent = Mock()
await self.dlmgr.start_handle(mock_download, {})
check_was_run.assert_called()
self.dlmgr.downloads.clear()
# Test waiting on DHT getting enough nodes
self.dlmgr.dht_readiness_timeout = 100
flag.append(True)
mock_download.future_added = succeed(True)
await self.dlmgr.start_handle(mock_download, {})
self.dlmgr.downloads.clear()
async def test_check_torrent_health(self):
"""
Test the endpoint to fetch the health of a chant-managed, infohash-only torrent
"""
infohash = b'a' * 20
tracker_url = 'udp://localhost:%s/announce' % self.udp_port
self.udp_tracker.tracker_info.add_info_about_infohash(
infohash, 12, 11, 1)
with db_session:
tracker_state = self.session.mds.TrackerState(url=tracker_url)
torrent_state = self.session.mds.TorrentState(
trackers=tracker_state, infohash=infohash)
self.session.mds.TorrentMetadata(infohash=infohash,
title='ubuntu-torrent.iso',
size=42,
tracker_info=tracker_url,
health=torrent_state)
url = 'metadata/torrents/%s/health?timeout=%s&refresh=1' % (
hexlify(infohash), TORRENT_CHECK_TIMEOUT)
# Initialize the torrent checker
self.session.torrent_checker = TorrentChecker(self.session)
await self.session.torrent_checker.initialize()
# Add mock DHT response - we both need to account for the case when BEP33 is used and the old lookup method
self.session.dlmgr = MockObject()
self.session.dlmgr.get_metainfo = lambda _, **__: succeed(None)
self.session.dlmgr.dht_health_manager = MockObject()
dht_health_dict = {
"infohash": hexlify(infohash),
"seeders": 1,
"leechers": 2
}
self.session.dlmgr.dht_health_manager.get_health = lambda *_, **__: succeed(
{"DHT": [dht_health_dict]})
self.session.dlmgr.get_channel_downloads = lambda: []
# Left for compatibility with other tests in this object
await self.udp_tracker.start()
json_response = await self.do_request(url)
self.assertIn("health", json_response)
self.assertIn("udp://localhost:%s" % self.udp_port,
json_response['health'])
if has_bep33_support():
self.assertIn("DHT", json_response['health'])
json_response = await self.do_request(url + '&nowait=1')
self.assertDictEqual(json_response, {'checking': '1'})
async def test_on_got_invalid_metainfo(self):
"""
Test whether the right operations happen when we receive an invalid metainfo object
"""
def get_metainfo(infohash, *_, **__):
return succeed("abcd")
self.session.dlmgr = Mock()
self.session.dlmgr.get_metainfo = get_metainfo
self.session.dlmgr.shutdown = lambda: succeed(None)
self.session.dlmgr.shutdown_downloads = lambda: succeed(None)
self.session.dlmgr.checkpoint_downloads = lambda: succeed(None)
path = 'magnet:?xt=urn:btih:%s&dn=%s' % (hexlify(UBUNTU_1504_INFOHASH),
quote_plus('test torrent'))
await self.do_request('torrentinfo?uri=%s' % path, expected_code=500)
def test_monitor_downloads_recreate_ip(self):
"""
Test whether an old introduction point is recreated
"""
mock_state = MockObject()
mock_download = MockObject()
mock_tdef = MockObject()
mock_tdef.get_infohash = lambda: b'a'
mock_download.get_def = lambda: mock_tdef
mock_download.add_peer = lambda _: succeed(None)
mock_download.get_state = lambda: mock_state
mock_download.config = MockObject()
mock_download.config.get_hops = lambda: 1
mock_download.apply_ip_filter = lambda _: None
mock_state.get_status = lambda: 4
mock_state.get_download = lambda: mock_download
def mock_create_ip(*_, **__):
mock_create_ip.called = True
mock_create_ip.called = False
self.nodes[0].overlay.create_introduction_point = mock_create_ip
self.nodes[0].overlay.download_states[b'a'] = 3
self.nodes[0].overlay.monitor_downloads([mock_state])
self.assertTrue(mock_create_ip.called)
async def test_get_metainfo_with_already_added_torrent(self):
"""
Testing metainfo fetching for a torrent which is already in session.
"""
sample_torrent = TESTS_DATA_DIR / "bak_single.torrent"
torrent_def = TorrentDef.load(sample_torrent)
download_impl = Mock()
download_impl.future_metainfo = succeed(bencode(torrent_def.get_metainfo()))
download_impl.checkpoint = lambda: succeed(None)
download_impl.stop = lambda: succeed(None)
download_impl.shutdown = lambda: succeed(None)
self.dlmgr.initialize()
self.dlmgr.downloads[torrent_def.infohash] = download_impl
self.assertTrue(await self.dlmgr.get_metainfo(torrent_def.infohash))
async def setUp(self):
await super(TestTorrentChecker, self).setUp()
self.session.torrent_checker = TorrentChecker(self.session)
self.session.tracker_manager = TrackerManager(self.session)
self.torrent_checker = self.session.torrent_checker
self.torrent_checker.listen_on_udp = lambda: succeed(None)
def get_metainfo(_, callback, **__):
callback({"seeders": 1, "leechers": 2})
self.session.dlmgr = Mock()
self.session.dlmgr.get_metainfo = get_metainfo
self.session.dlmgr.shutdown = lambda: succeed(None)
self.session.dlmgr.shutdown_downloads = lambda: succeed(None)
self.session.dlmgr.checkpoint_downloads = lambda: succeed(None)
def connect_peer(self, mid):
matched_node = MockObject()
matched_node.mid = mid
matched_node.public_key = ECCrypto().generate_key("low").pub()
nearby_node = MockObject()
nearby_node.mid = unhexlify('b' * 20)
nearby_node.public_key = ECCrypto().generate_key("low").pub()
return succeed([matched_node, nearby_node])
async def test_connect_to_tracker_fail(self):
"""
Test the metainfo lookup of the DHT session when it fails
"""
self.session.dlmgr.get_metainfo = lambda *_, **__: succeed(None)
try:
await self.dht_session.connect_to_tracker()
except:
pass
async def test_get_metainfo_valid_metadata(self):
"""
Testing the get_metainfo method when the handle has valid metadata immediately
"""
infohash = b"a" * 20
metainfo = {b'info': {b'pieces': [b'a']}, b'leechers': 0,
b'nodes': [], b'seeders': 0}
download_impl = Mock()
download_impl.tdef.get_metainfo = lambda: None
download_impl.future_metainfo = succeed(metainfo)
self.dlmgr.initialize()
self.dlmgr.start_download = Mock(return_value=download_impl)
self.dlmgr.tribler_session.config.get_default_number_hops = lambda: 1
self.dlmgr.remove_download = Mock(return_value=succeed(None))
self.assertEqual(await self.dlmgr.get_metainfo(infohash), metainfo)
self.dlmgr.start_download.assert_called_once()
self.dlmgr.remove_download.assert_called_once()
async def test_get_metainfo_add_fail(self):
"""
Test whether we try to add a torrent again if the atp is rejected
"""
infohash = b"a" * 20
metainfo = {'pieces': ['a']}
download_impl = Mock()
download_impl.future_metainfo = succeed(metainfo)
download_impl.tdef.get_metainfo = lambda: None
self.dlmgr.initialize()
self.dlmgr.start_download = Mock()
self.dlmgr.start_download.side_effect = TypeError
self.dlmgr.tribler_session.config.get_default_number_hops = lambda: 1
self.dlmgr.remove = Mock(return_value=succeed(None))
self.assertEqual(await self.dlmgr.get_metainfo(infohash), None)
self.dlmgr.start_download.assert_called_once()
self.dlmgr.remove.assert_not_called()
def get_metainfo(infohash, timeout=20, hops=None, url=None):
if hops is not None:
hops_list.append(hops)
with open(TESTS_DATA_DIR /
"ubuntu-15.04-desktop-amd64.iso.torrent",
mode='rb') as torrent_file:
torrent_data = torrent_file.read()
tdef = TorrentDef.load_from_memory(torrent_data)
self.assertIsNotNone(url)
self.assertEqual(url, unquote_plus(path))
return succeed(tdef.get_metainfo())
def remove_circuit(self,
circuit_id,
additional_info='',
remove_now=False,
destroy=False):
if circuit_id not in self.circuits:
self.logger.warning(
"Circuit %d not found when trying to remove it", circuit_id)
return succeed(None)
circuit = self.circuits[circuit_id]
# Send the notification
if self.tribler_session:
self.tribler_session.notifier.notify(NTFY.TUNNEL_REMOVE, circuit,
additional_info)
# Ignore circuits that are closing so we do not payout again if we receive a destroy message.
if circuit.state != CIRCUIT_STATE_CLOSING and circuit.bytes_down >= 1024 * 1024 and self.bandwidth_wallet:
# We should perform a payout of the removed circuit.
if circuit.ctype == CIRCUIT_TYPE_RP_DOWNLOADER:
# We remove an e2e circuit as downloader. We pay the subsequent nodes in the downloader part of the e2e
# circuit. In addition, we pay for one hop seeder anonymity since we don't know the circuit length at
# the seeder side.
self.do_payout(
circuit.peer, circuit_id,
circuit.bytes_down * ((circuit.goal_hops * 2) + 1),
circuit.bytes_down)
if circuit.ctype == CIRCUIT_TYPE_DATA:
# We remove a regular data circuit as downloader. Pay the relay nodes and the exit nodes.
self.do_payout(
circuit.peer, circuit_id,
circuit.bytes_down * (circuit.goal_hops * 2 - 1),
circuit.bytes_down)
affected_peers = self.dispatcher.circuit_dead(circuit)
# Make sure the circuit is marked as closing, otherwise we may end up reusing it
circuit.close()
if self.tribler_session and self.tribler_session.config.get_libtorrent_enabled(
):
for download in self.tribler_session.dlmgr.get_downloads():
self.update_torrent(affected_peers, download)
# Now we actually remove the circuit
return super(TriblerTunnelCommunity,
self).remove_circuit(circuit_id,
additional_info=additional_info,
remove_now=remove_now,
destroy=destroy)
def test_readd_bittorrent_peers(self):
"""
Test the readd bittorrent peers method
"""
mock_torrent = MockObject()
mock_torrent.add_peer = lambda _: succeed(None)
mock_torrent.tdef = MockObject()
mock_torrent.tdef.get_infohash = lambda: b'a' * 20
self.nodes[0].overlay.bittorrent_peers = {mock_torrent: [None]}
self.nodes[0].overlay.readd_bittorrent_peers()
self.assertNotIn(mock_torrent, self.nodes[0].overlay.bittorrent_peers)
async def test_connect_to_tracker(self):
"""
Test the metainfo lookup of the DHT session
"""
metainfo = {b'seeders': 42, b'leechers': 42}
self.session.dlmgr.get_metainfo = lambda *_, **__: succeed(metainfo)
metainfo = await self.dht_session.connect_to_tracker()
self.assertTrue('DHT' in metainfo)
self.assertEqual(metainfo['DHT'][0]['leechers'], 42)
self.assertEqual(metainfo['DHT'][0]['seeders'], 42)
async def setUp(self):
await super(TestPayoutManager, self).setUp()
fake_tc = MockObject()
fake_tc.add_listener = lambda *_: None
fake_response_peer = MockObject()
fake_response_peer.public_key = MockObject()
fake_response_peer.public_key.key_to_bin = lambda: b'a' * 64
fake_dht = MockObject()
fake_dht.connect_peer = lambda *_: succeed([fake_response_peer])
self.payout_manager = PayoutManager(fake_tc, fake_dht)
def should_join_circuit(self, create_payload, previous_node_address):
"""
Check whether we should join a circuit. Returns a future that fires with a boolean.
"""
if self.settings.max_joined_circuits <= len(self.relay_from_to) + len(
self.exit_sockets):
self.logger.warning(
"too many relays (%d)",
(len(self.relay_from_to) + len(self.exit_sockets)))
return succeed(False)
# Check whether we have a random open slot, if so, allocate this to this request.
circuit_id = create_payload.circuit_id
for index, slot in enumerate(self.random_slots):
if not slot:
self.random_slots[index] = circuit_id
return succeed(True)
# No random slots but this user might be allocated a competing slot.
# Next, we request the token balance of the circuit initiator.
balance_future = Future()
self.request_cache.add(
BalanceRequestCache(self, circuit_id, balance_future))
# Temporarily add these values, otherwise we are unable to communicate with the previous hop.
self.directions[circuit_id] = EXIT_NODE
shared_secret, _, _ = self.crypto.generate_diffie_shared_secret(
create_payload.key)
self.relay_session_keys[
circuit_id] = self.crypto.generate_session_keys(shared_secret)
self.send_cell(
Peer(create_payload.node_public_key, previous_node_address),
"balance-request", BalanceRequestPayload(circuit_id))
self.directions.pop(circuit_id, None)
self.relay_session_keys.pop(circuit_id, None)
return balance_future
def mock_endpoints(self):
self.session.api_manager = MockObject()
self.session.api_manager.stop = lambda: succeed(None)
mocked_endpoints = {}
def get_endpoint_mock(name):
if name in mocked_endpoints:
return mocked_endpoints[name]
endpoint = Mock()
mocked_endpoints[name] = endpoint
return endpoint
self.session.api_manager.get_endpoint = get_endpoint_mock
async def test_multifile_torrent(self):
# Achtung! This test is completely and utterly broken, as is the whole libtorrent wrapper!
# Don't try to understand it, it is a legacy thing!
tdef = TorrentDef()
tdef.add_content(TESTS_DATA_DIR / "video.avi")
tdef.set_tracker("http://tribler.org/announce")
tdef.save()
fake_handler = MockObject()
fake_handler.is_valid = lambda: True
fake_handler.status = lambda: fake_status
fake_handler.set_share_mode = lambda _: None
fake_handler.set_priority = lambda _: None
fake_handler.set_sequential_download = lambda _: None
fake_handler.resume = lambda: None
fake_handler.set_max_connections = lambda _: None
fake_handler.apply_ip_filter = lambda _: None
fake_handler.save_resume_data = lambda: None
fake_status = MockObject()
fake_status.share_mode = False
dl = Download(self.session, tdef)
dl.set_selected_files = lambda: None
dl.future_added = succeed(fake_handler)
# Create a dummy download config
dl.config = DownloadConfig()
dl.config.set_engineresumedata({
b"save_path":
path_util.abspath(self.state_dir),
b"info-hash":
b'\x00' * 20
})
dl.setup()
dl.config.set_engineresumedata({
b"save_path":
path_util.abspath(self.state_dir),
b"info-hash":
b'\x00' * 20
})
dl.setup()
dl.config.set_engineresumedata({
b"save_path": "some_local_dir",
b"info-hash": b'\x00' * 20
})
dl.setup()
await dl.shutdown()
async def setUp(self):
await super(TestDHTSession, self).setUp()
self.session = Session(TriblerConfig(self.root_state_dir))
self.session.dlmgr = MockObject()
self.session.dlmgr.dht_health_manager = MockObject()
dht_health_dict = {
"infohash": hexlify(b'a' * 20),
"seeders": 1,
"leechers": 2
}
self.session.dlmgr.dht_health_manager.get_health = lambda *_, **__: succeed(
{"DHT": [dht_health_dict]})
self.dht_session = FakeDHTSession(self.session, b'a' * 20, 10)
self.bep33_dht_session = FakeBep33DHTSession(self.session, b'a' * 20,
10)
async def test_task_select_tracker(self):
with db_session:
tracker = self.session.mds.TrackerState(
url="http://localhost/tracker")
self.session.mds.TorrentState(infohash=b'a' * 20,
seeders=5,
leechers=10,
trackers={tracker})
controlled_session = HttpTrackerSession("127.0.0.1",
("localhost", 8475),
"/announce", 5)
controlled_session.connect_to_tracker = lambda: succeed(None)
self.torrent_checker._create_session_for_request = lambda *args, **kwargs: controlled_session
result = await self.torrent_checker.check_random_tracker()
self.assertFalse(result)
self.assertEqual(len(controlled_session.infohash_list), 1)
def test_check_random_torrent(self):
"""
Test that the random torrent health checking mechanism picks the right torrents
"""
for ind in range(1, 20):
torrent = self.session.mds.TorrentMetadata(title='torrent1',
infohash=os.urandom(20))
torrent.health.last_check = ind
self.torrent_checker.check_torrent_health = lambda _: succeed(None)
random_infohashes = self.torrent_checker.check_random_torrent()
self.assertTrue(random_infohashes)
# Now we should only check a single torrent
self.torrent_checker.torrents_checked.add(
(b'a' * 20, 5, 5, int(time.time())))
random_infohashes = self.torrent_checker.check_random_torrent()
self.assertEqual(len(random_infohashes), 1)
async def test_get_metainfo_duplicate_request(self):
"""
Test whether the same request is returned when invoking get_metainfo twice with the same infohash
"""
infohash = b"a" * 20
metainfo = {'pieces': ['a']}
download_impl = Mock()
download_impl.tdef.get_metainfo = lambda: None
download_impl.future_metainfo = Future()
get_event_loop().call_later(0.1, download_impl.future_metainfo.set_result, metainfo)
self.dlmgr.initialize()
self.dlmgr.start_download = Mock(return_value=download_impl)
self.dlmgr.tribler_session.config.get_default_number_hops = lambda: 1
self.dlmgr.remove_download = Mock(return_value=succeed(None))
results = await gather(self.dlmgr.get_metainfo(infohash), self.dlmgr.get_metainfo(infohash))
self.assertEqual(results, [metainfo, metainfo])
self.dlmgr.start_download.assert_called_once()
self.dlmgr.remove_download.assert_called_once()
async def setUp(self):
await super(TestTrustViewEndpoint, self).setUp()
self.mock_ipv8 = MockIPv8(
u"low",
TrustChainCommunity,
working_directory=self.session.config.get_state_dir())
self.session.trustchain_community = self.mock_ipv8.overlay
self.session.bootstrap = Mock()
self.session.bootstrap.shutdown = lambda: succeed(None)
bootstrap_download_state = MockObject()
bootstrap_download_state.get_total_transferred = lambda _: random.randint(
0, 10000)
bootstrap_download_state.get_progress = lambda: random.randint(10, 100)
self.session.bootstrap.download.get_state = lambda: bootstrap_download_state
self.endpoint = self.session.api_manager.root_endpoint.endpoints[
'/trustview']
async def test_start_download_while_getting_metainfo(self):
"""
Testing adding a torrent while a metainfo request is running.
"""
infohash = b"a" * 20
metainfo_session = Mock()
metainfo_session.get_torrents = lambda: []
metainfo_dl = Mock()
metainfo_dl.get_def = lambda: Mock(get_infohash=lambda: infohash)
self.dlmgr.initialize()
self.dlmgr.get_session = lambda *_: metainfo_session
self.dlmgr.downloads[infohash] = metainfo_dl
self.dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1]
self.dlmgr.remove_download = Mock(return_value=succeed(None))
tdef = TorrentDefNoMetainfo(infohash, 'name', 'magnet:?xt=urn:btih:%s&' % hexlify(infohash))
download = self.dlmgr.start_download(tdef=tdef, checkpoint_disabled=True)
self.assertNotEqual(metainfo_dl, download)
await sleep(.1)
self.assertEqual(self.dlmgr.downloads[infohash], download)
self.dlmgr.remove_download.assert_called_once_with(metainfo_dl, remove_content=True, remove_checkpoint=False)
def fake_shutdown():
# Record session.shutdown was called
self.shutdown_called = True
# Restore original shutdown for test teardown
self.session.shutdown = self.orig_shutdown
return succeed(True)
async def setUp(self):
await super(TestSpecificChannelEndpoint, self).setUp()
self.session.dlmgr = Mock()
self.session.dlmgr.shutdown = lambda: succeed(True)
self.session.dlmgr.shutdown_downloads = lambda: succeed(True)
self.session.dlmgr.checkpoint_downloads = lambda: succeed(True)
def fake_get_metainfo(*_, **__):
return succeed(None)
def fake_get_metainfo(_, **__):
meta_info = TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo()
return succeed(meta_info)
async def setUp(self):
await super(BaseTestMyChannelEndpoint, self).setUp()
self.session.gigachannel_manager = MockObject()
self.session.gigachannel_manager.shutdown = lambda: succeed(None)
self.session.gigachannel_manager.updated_my_channel = lambda _: succeed(None)
