async def test_load_from_url(file_server, tmpdir): shutil.copyfile(TORRENT_UBUNTU_FILE, tmpdir / 'ubuntu.torrent') torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server torrent_def = await TorrentDef.load_from_url(torrent_url) assert torrent_def.get_metainfo() == TorrentDef.load( TORRENT_UBUNTU_FILE).get_metainfo() assert torrent_def.infohash == TorrentDef.load( TORRENT_UBUNTU_FILE).infohash
def test_is_private(): """ Test whether the private field from an existing torrent is correctly read """ privatefn = TESTS_DATA_DIR / "private.torrent" publicfn = TESTS_DATA_DIR / "bak_single.torrent" t1 = TorrentDef.load(privatefn) t2 = TorrentDef.load(publicfn) assert t1.is_private() assert not t2.is_private()
def test_is_private(self): """ Test whether the private field from an existing torrent is correctly read """ privatefn = TESTS_DATA_DIR / "private.torrent" publicfn = TESTS_DATA_DIR / "bak_single.torrent" t1 = TorrentDef.load(privatefn) t2 = TorrentDef.load(publicfn) self.assertTrue(t1.is_private()) self.assertFalse(t2.is_private())
def test_commit_channel_torrent(self): channel = self.mds.ChannelMetadata.create_channel('test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel.add_torrent_to_channel(tdef, None) # The first run should return the infohash, the second should return None, because nothing was really done self.assertTrue(channel.commit_channel_torrent()) self.assertFalse(channel.commit_channel_torrent())
def test_restore_torrent_in_channel(self): """ Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it. """ channel_metadata = self.mds.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) md = channel_metadata.add_torrent_to_channel(tdef, None) # Check correct re-add md.status = TODELETE md_updated = channel_metadata.add_torrent_to_channel(tdef, None) self.assertEqual(UPDATED, md.status) self.assertEqual(md_updated, md) self.assertTrue(md.has_valid_signature) # Check update of torrent properties from a new tdef md.status = TODELETE new_tracker_address = u'http://tribler.org/announce' tdef.torrent_parameters[b'announce'] = new_tracker_address.encode( 'utf-8') md_updated = channel_metadata.add_torrent_to_channel(tdef, None) self.assertEqual(md_updated, md) self.assertEqual(md.status, UPDATED) self.assertEqual(md.tracker_info, new_tracker_address) self.assertTrue(md.has_valid_signature) # In addition, check that the trackers table was properly updated self.assertEqual(len(md.health.trackers), 2)
def test_restore_torrent_in_channel(metadata_store): """ Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it. """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) md = channel_metadata.add_torrent_to_channel(tdef, None) # Check correct re-add md.status = TODELETE md_updated = channel_metadata.add_torrent_to_channel(tdef, None) assert UPDATED == md.status assert md_updated == md assert md.has_valid_signature # Check update of torrent properties from a new tdef md.status = TODELETE new_tracker_address = 'http://tribler.org/announce' tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8') md_updated = channel_metadata.add_torrent_to_channel(tdef, None) assert md_updated == md assert md.status == UPDATED assert md.tracker_info == new_tracker_address assert md.has_valid_signature # In addition, check that the trackers table was properly updated assert len(md.health.trackers) == 2
def check_watch_folder(self): if not self.session.config.get_watch_folder_path().is_dir(): return # Make sure that we pass a str to os.walk watch_dir = str(self.session.config.get_watch_folder_path()) for root, _, files in os.walk(watch_dir): root = path_util.Path(root) for name in files: if not name.endswith(".torrent"): continue try: tdef = TorrentDef.load(root / name) if not tdef.get_metainfo(): self.cleanup_torrent_file(root, name) continue except: # torrent appears to be corrupt self.cleanup_torrent_file(root, name) continue infohash = tdef.get_infohash() if not self.session.dlmgr.download_exists(infohash): self._logger.info("Starting download from torrent file %s", name) dl_config = DownloadConfig() anon_enabled = self.session.config.get_default_anonymity_enabled() default_num_hops = self.session.config.get_default_number_hops() dl_config.set_hops(default_num_hops if anon_enabled else 0) dl_config.set_safe_seeding(self.session.config.get_default_safeseeding_enabled()) dl_config.set_dest_dir(self.session.config.get_default_destination_dir()) self.session.dlmgr.start_download(tdef=tdef, config=dl_config)
async def test_create_torrent(self): """ Testing whether the API returns a proper base64 encoded torrent """ torrent_path = self.files_path / "video.avi.torrent" expected_tdef = TorrentDef.load(torrent_path) export_dir = self.temporary_directory() post_data = { "files": [self.files_path / "video.avi", self.files_path / "video.avi.torrent"], "description": "Video of my cat", "trackers": "http://localhost/announce", "name": "test_torrent", "export_dir": export_dir } response_dict = await self.do_request('createtorrent?download=1', 200, None, 'POST', post_data) torrent = base64.b64decode(response_dict["torrent"]) tdef = TorrentDef.load_from_memory(torrent) # Copy expected creation date and created by (Tribler version) from actual result creation_date = tdef.get_creation_date() expected_tdef.metainfo[b"creation date"] = creation_date expected_tdef.metainfo[b"created by"] = tdef.metainfo[b'created by'] self.assertEqual(dir(expected_tdef), dir(tdef)) self.assertTrue((export_dir / "test_torrent.torrent").exists())
def add_torrents_from_dir(self, torrents_dir, recursive=False): torrents_list = [] errors_list = [] def rec_gen(dir_): for root, _, filenames in os.walk(dir_): for fn in filenames: yield Path(root) / fn filename_generator = rec_gen(torrents_dir) if recursive else os.listdir(torrents_dir) # Build list of .torrents to process torrents_list_generator = (Path(torrents_dir, f) for f in filename_generator) torrents_list = [f for f in torrents_list_generator if f.is_file() and f.suffix == ".torrent"] # 100 is a reasonable chunk size for commits for chunk in chunks(torrents_list, 100): for f in chunk: try: self.add_torrent_to_channel(TorrentDef.load(f)) except DuplicateTorrentFileError: pass except Exception: # pylint: disable=W0703 # Have to use the broad exception clause because Py3 versions of libtorrent # generate generic Exceptions errors_list.append(f) # Optimization to drop excess cache orm.commit() return torrents_list, errors_list
async def test_load_from_url(self): # Setup file server to serve torrent file self.session_base_dir = mkdtemp(suffix="_tribler_test_load_from_url") files_path = self.session_base_dir / 'http_torrent_files' os.mkdir(files_path) shutil.copyfile(TORRENT_UBUNTU_FILE, files_path / 'ubuntu.torrent') file_server_port = self.get_port() await self.setUpFileServer(file_server_port, files_path) torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server_port torrent_def = await TorrentDef.load_from_url(torrent_url) self.assertEqual(torrent_def.get_metainfo(), TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo()) self.assertEqual(torrent_def.infohash, TorrentDef.load(TORRENT_UBUNTU_FILE).infohash)
def personal_channel(session): global update_metainfo with db_session: chan = session.mds.ChannelMetadata.create_channel(title="my test chan", description="test") tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) chan.add_torrent_to_channel(tdef, None) update_metainfo = chan.commit_channel_torrent() return chan
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False): self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef) # the priority of the parameters is: (1) tdef, (2) torrent_file. # so if we have tdef, and torrent_file will be ignored, and so on. if tdef is None: if torrent_file is None: raise ValueError( "Torrent file must be provided if tdef is not given") # try to get the torrent from the given torrent file tdef = TorrentDef.load(torrent_file) assert tdef is not None, "tdef MUST not be None after loading torrent" config = config or DownloadConfig() infohash = tdef.get_infohash() download = self.get_download(infohash) if download and infohash not in self.metainfo_requests: new_trackers = list( set(tdef.get_trackers_as_single_tuple()) - set(download.get_def().get_trackers_as_single_tuple())) if new_trackers: self.update_trackers(tdef.get_infohash(), new_trackers) return download # Create the destination directory if it does not exist yet try: if not config.get_dest_dir().is_dir(): os.makedirs(config.get_dest_dir()) except OSError: self._logger.error( "Unable to create the download destination directory.") if config.get_time_added() == 0: config.set_time_added(int(timemod.time())) # Create the download download = Download(self.tribler_session, tdef, dummy=self.dummy_mode) atp = download.setup(config, checkpoint_disabled=checkpoint_disabled, hidden=hidden or config.get_bootstrap_download()) # Keep metainfo downloads in self.downloads for now because we will need to remove it later, # and removing the download at this point will stop us from receiving any further alerts. if infohash not in self.metainfo_requests or self.metainfo_requests[ infohash][0] == download: self.downloads[infohash] = download if not self.dummy_mode: self.start_handle(download, atp) return download
def test_commit_channel_torrent(metadata_store): """ Test committing a channel torrent """ channel = metadata_store.ChannelMetadata.create_channel('test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel.add_torrent_to_channel(tdef, None) # The first run should return the infohash, the second should return None, because nothing was really done assert channel.commit_channel_torrent() assert not channel.commit_channel_torrent()
def test_add_torrent_to_channel(metadata_store): """ Test adding a torrent to your channel """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'}) assert channel_metadata.contents_list with pytest.raises(DuplicateTorrentFileError): channel_metadata.add_torrent_to_channel(tdef, None)
async def test_channel_update_and_download(self): """ Test whether we can successfully update a channel and download the new version """ # First we have to manually add the old version old_payload = ChannelMetadataPayload.from_file(CHANNEL_METADATA) with db_session: old_channel = self.session.mds.ChannelMetadata.from_payload( old_payload) chan_dir = CHANNEL_DIR / old_channel.dirname self.session.mds.process_channel_dir(chan_dir, old_payload.public_key, old_payload.id_) channel_tdef = TorrentDef.load(CHANNEL_TORRENT_UPDATED) libtorrent_port = self.get_port() await self.setup_seeder(channel_tdef, CHANNEL_DIR, libtorrent_port) payload = ChannelMetadataPayload.from_file(CHANNEL_METADATA_UPDATED) # Download the channel in our session with db_session: self.session.mds.process_payload(payload) channel = self.session.mds.ChannelMetadata.get( signature=payload.signature) def fake_get_metainfo(*args, **kwargs): return succeed(channel_tdef.get_metainfo()) self.session.dlmgr.get_metainfo = fake_get_metainfo # The leecher should be hinted to leech from localhost. Thus, we must extend start_download_from_tdef # and get_metainfo to provide the hint. original_start_download_from_tdef = self.session.dlmgr.start_download def hinted_start_download(tdef=None, config=None, hidden=False): download = original_start_download_from_tdef(tdef=tdef, config=config, hidden=hidden) download.add_peer( ("127.0.0.1", self.seeder_session.config.get_libtorrent_port())) return download self.session.dlmgr.start_download = hinted_start_download await self.session.gigachannel_manager.download_channel(channel) await self.session.gigachannel_manager.process_queued_channels() with db_session: # There should be 8 torrents + 1 channel torrent channel2 = self.session.mds.ChannelMetadata.get( public_key=database_blob(payload.public_key)) self.assertEqual(channel2.timestamp, channel2.local_version) self.assertEqual(1565621688018, channel2.timestamp) self.assertEqual(8, self.session.mds.ChannelNode.select().count())
def test_add_torrent_to_channel(self): """ Test adding a torrent to your channel """ channel_metadata = self.mds.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'}) self.assertTrue(channel_metadata.contents_list) self.assertRaises(DuplicateTorrentFileError, channel_metadata.add_torrent_to_channel, tdef, None)
def gen_sample_channel(mds): my_channel = mds.ChannelMetadata.create_channel('test_channel', 'test description') my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_UBUNTU_FILE), None) my_channel.commit_channel_torrent() t2 = my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_VIDEO_FILE), None) mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) coll = mds.CollectionNode(origin_id=my_channel.id_, title='internal collection') mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) my_channel.commit_channel_torrent() t2.soft_delete() my_channel.commit_channel_torrent() # Rename files to stable names mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob") torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent") os.rename(mdblob_name, CHANNEL_METADATA) os.rename(torrent_name, CHANNEL_TORRENT) # Update channel mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) my_channel.commit_channel_torrent() # Rename updated files to stable names os.rename(mdblob_name, CHANNEL_METADATA_UPDATED) os.rename(torrent_name, CHANNEL_TORRENT_UPDATED)
def test_create_ffa_from_dict(metadata_store): """ Test creating a free-for-all torrent entry """ tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created signed_entry = metadata_store.TorrentMetadata.from_dict(tdef_to_metadata_dict(tdef)) metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 0 signed_entry.delete() # Create FFA entry metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 1
async def test_get_metainfo_with_already_added_torrent(fake_dlmgr): """ Testing metainfo fetching for a torrent which is already in session. """ sample_torrent = TESTS_DATA_DIR / "bak_single.torrent" torrent_def = TorrentDef.load(sample_torrent) download_impl = Mock() download_impl.future_metainfo = succeed(bencode(torrent_def.get_metainfo())) download_impl.checkpoint = lambda: succeed(None) download_impl.stop = lambda: succeed(None) download_impl.shutdown = lambda: succeed(None) fake_dlmgr.initialize() fake_dlmgr.downloads[torrent_def.infohash] = download_impl assert await fake_dlmgr.get_metainfo(torrent_def.infohash)
def test_consolidate_channel_torrent(torrent_template, metadata_store): """ Test completely re-commit your channel """ channel = metadata_store.ChannelMetadata.create_channel('test', 'test') my_dir = path_util.abspath(metadata_store.ChannelMetadata._channels_dir / channel.dirname) tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # 1st torrent torrent_entry = channel.add_torrent_to_channel(tdef, None) channel.commit_channel_torrent() # 2nd torrent metadata_store.TorrentMetadata.from_dict( dict(torrent_template, public_key=channel.public_key, origin_id=channel.id_, status=NEW)) channel.commit_channel_torrent() # Delete entry torrent_entry.soft_delete() channel.commit_channel_torrent() assert len(channel.contents_list) == 1 assert len(os.listdir(my_dir)) == 3 torrent3 = metadata_store.TorrentMetadata(public_key=channel.public_key, origin_id=channel.id_, status=NEW, infohash=random_infohash()) channel.commit_channel_torrent() torrent3.soft_delete() channel.consolidate_channel_torrent() assert len(os.listdir(my_dir)) == 1 metadata_store.TorrentMetadata.select( lambda g: g.metadata_type == REGULAR_TORRENT).delete() channel.local_version = 0 metadata_store.process_channel_dir(my_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) assert len(channel.contents[:]) == 1
async def test_add_torrent_duplicate(enable_chant, enable_api, my_channel, session): """ Test whether adding a duplicate torrent to you channel results in an error """ with db_session: tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'}) with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file: base64_content = base64.b64encode(torrent_file.read()).decode('utf-8') post_params = {'torrent': base64_content} await do_request( session, f'channels/{hexlify(my_channel.public_key)}/{my_channel.id_}/torrents', request_type='PUT', post_data=post_params, expected_code=500, )
async def test_add_torrent_duplicate(self): """ Test whether adding a duplicate torrent to you channel results in an error """ with db_session: channel = self.create_my_channel() my_channel = self.session.mds.ChannelMetadata.get_my_channels().first() tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'}) with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file: base64_content = base64.b64encode(torrent_file.read()).decode('utf-8') post_params = {'torrent': base64_content} await self.do_request( 'channels/%s/%s/torrents' % (hexlify(channel.public_key), channel.id_), request_type='PUT', post_data=post_params, expected_code=500, )
def test_commit_channel_torrent(metadata_store): """ Test committing a channel torrent """ channel = metadata_store.ChannelMetadata.create_channel('test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel.add_torrent_to_channel(tdef, None) # The first run should return the infohash, the second should return None, because nothing was really done assert channel.commit_channel_torrent() assert not channel.commit_channel_torrent() # Test adding flags to channel torrent when adding thumbnail and description metadata_store.ChannelThumbnail(public_key=channel.public_key, origin_id=channel.id_, status=NEW) metadata_store.ChannelDescription(public_key=channel.public_key, origin_id=channel.id_, status=NEW) assert channel.commit_channel_torrent() assert channel.reserved_flags == 3 assert not channel.commit_channel_torrent()
def test_delete_torrent_from_channel(metadata_store): """ Test deleting a torrent from your channel """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # Check that nothing is committed when deleting uncommited torrent metadata torrent = channel_metadata.add_torrent_to_channel(tdef, None) torrent.soft_delete() assert not channel_metadata.contents_list # Check append-only deletion process torrent = channel_metadata.add_torrent_to_channel(tdef, None) channel_metadata.commit_channel_torrent() assert len(channel_metadata.contents_list) == 1 torrent.soft_delete() channel_metadata.commit_channel_torrent() assert not channel_metadata.contents_list
def fake_get_metainfo(_, **__): meta_info = TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo() return succeed(meta_info)
async def get_torrent_info(self, request): args = request.query hops = None if 'hops' in args: try: hops = int(args['hops']) except ValueError: return RESTResponse( { "error": f"wrong value of 'hops' parameter: {repr(args['hops'])}" }, status=HTTP_BAD_REQUEST) if 'uri' not in args or not args['uri']: return RESTResponse({"error": "uri parameter missing"}, status=HTTP_BAD_REQUEST) uri = args['uri'] if uri.startswith('file:'): try: filename = url2pathname(uri[5:]) tdef = TorrentDef.load(filename) metainfo = tdef.get_metainfo() except (TypeError, RuntimeError): return RESTResponse( {"error": "error while decoding torrent file"}, status=HTTP_INTERNAL_SERVER_ERROR) elif uri.startswith('http'): try: async with ClientSession(raise_for_status=True) as session: response = await session.get(uri) response = await response.read() except (ServerConnectionError, ClientResponseError) as e: return RESTResponse({"error": str(e)}, status=HTTP_INTERNAL_SERVER_ERROR) if response.startswith(b'magnet'): _, infohash, _ = parse_magnetlink(response) if infohash: metainfo = await self.session.dlmgr.get_metainfo( infohash, timeout=60, hops=hops, url=response) else: metainfo = bdecode_compat(response) elif uri.startswith('magnet'): infohash = parse_magnetlink(uri)[1] if infohash is None: return RESTResponse({"error": "missing infohash"}, status=HTTP_BAD_REQUEST) metainfo = await self.session.dlmgr.get_metainfo(infohash, timeout=60, hops=hops, url=uri) else: return RESTResponse({"error": "invalid uri"}, status=HTTP_BAD_REQUEST) if not metainfo: return RESTResponse({"error": "metainfo error"}, status=HTTP_INTERNAL_SERVER_ERROR) if not isinstance(metainfo, dict) or b'info' not in metainfo: self._logger.warning("Received metainfo is not a valid dictionary") return RESTResponse({"error": "invalid response"}, status=HTTP_INTERNAL_SERVER_ERROR) # Add the torrent to GigaChannel as a free-for-all entry, so others can search it self.session.mds.TorrentMetadata.add_ffa_from_dict( tdef_to_metadata_dict(TorrentDef.load_from_dict(metainfo))) # TODO(Martijn): store the stuff in a database!!! # TODO(Vadim): this means cache the downloaded torrent in a binary storage, like LevelDB infohash = hashlib.sha1(bencode(metainfo[b'info'])).digest() download = self.session.dlmgr.downloads.get(infohash) metainfo_request = self.session.dlmgr.metainfo_requests.get( infohash, [None])[0] download_is_metainfo_request = download == metainfo_request # Check if the torrent is already in the downloads encoded_metainfo = deepcopy(metainfo) # FIXME: json.dumps garbles binary data that is used by the 'pieces' field # However, this is fine as long as the GUI does not use this field. encoded_metainfo[b'info'][b'pieces'] = hexlify( encoded_metainfo[b'info'][b'pieces']).encode('utf-8') encoded_metainfo = hexlify( json.dumps(recursive_unicode(encoded_metainfo, ignore_errors=True), ensure_ascii=False).encode('utf-8')) return RESTResponse({ "metainfo": encoded_metainfo, "download_exists": download and not download_is_metainfo_request })
def channel_tdef(): return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' / 'channel_upd.torrent')
def video_tdef(): return TorrentDef.load(TESTS_DATA_DIR / 'video.avi.torrent')
async def setUp(self): await super(TestSeeding, self).setUp() self.tdef = TorrentDef.load(TESTS_DATA_DIR / 'video.avi.torrent') self.sourcefn = TESTS_DATA_DIR / 'video.avi'
def test_sanitize_tdef(metadata_store): tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) tdef.metainfo["creation date"] = -100000 assert metadata_store.TorrentMetadata.from_dict( tdef_to_metadata_dict(tdef))