def create_fake_download_and_state(): """ Create a fake download and state which can be passed to the global download callback. """ tdef = TorrentDef() tdef.get_infohash = lambda: b'aaaa' fake_peer = { 'extended_version': 'Tribler', 'id': 'a' * 20, 'dtotal': 10 * 1024 * 1024 } fake_download = MagicMock() fake_download.get_def = lambda: tdef fake_download.get_def().get_name_as_unicode = lambda: "test.iso" fake_download.get_peerlist = lambda: [fake_peer] fake_download.hidden = False fake_download.checkpoint = lambda: succeed(None) fake_download.stop = lambda: succeed(None) fake_download.shutdown = lambda: succeed(None) dl_state = MagicMock() dl_state.get_infohash = lambda: b'aaaa' dl_state.get_status = lambda: DLSTATUS_SEEDING dl_state.get_download = lambda: fake_download fake_config = MagicMock() fake_config.get_hops = lambda: 0 fake_config.get_safe_seeding = lambda: True fake_download.config = fake_config return fake_download, dl_state
async def test_load_from_url(file_server, tmpdir): shutil.copyfile(TORRENT_UBUNTU_FILE, tmpdir / 'ubuntu.torrent') torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server torrent_def = await TorrentDef.load_from_url(torrent_url) assert torrent_def.get_metainfo() == TorrentDef.load( TORRENT_UBUNTU_FILE).get_metainfo() assert torrent_def.infohash == TorrentDef.load( TORRENT_UBUNTU_FILE).infohash
def test_create_invalid_tdef(): """ Test whether creating invalid TorrentDef objects result in ValueErrors """ invalid_metainfo = {} with pytest.raises(ValueError): TorrentDef.load_from_memory(bencode(invalid_metainfo)) invalid_metainfo = {b'info': {}} with pytest.raises(ValueError): TorrentDef.load_from_memory(bencode(invalid_metainfo))
def test_is_private(): """ Test whether the private field from an existing torrent is correctly read """ privatefn = TESTS_DATA_DIR / "private.torrent" publicfn = TESTS_DATA_DIR / "bak_single.torrent" t1 = TorrentDef.load(privatefn) t2 = TorrentDef.load(publicfn) assert t1.is_private() assert not t2.is_private()
def test_tdef(state_dir): tdef = TorrentDef() sourcefn = TESTS_DATA_DIR / 'video.avi' tdef.add_content(sourcefn) tdef.set_tracker("http://localhost/announce") torrentfn = state_dir / "gen.torrent" tdef.save(torrentfn) return tdef
async def post_commit(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: if channel_id == 0: for t in self.mds.CollectionNode.commit_all_channels(): self.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t)) else: coll = self.mds.CollectionNode.get(public_key=channel_pk, id_=channel_id) if not coll: return RESTResponse({"success": False}, status=HTTP_NOT_FOUND) torrent_dict = coll.commit_channel_torrent() if torrent_dict: self.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(torrent_dict)) return RESTResponse({"success": True})
async def regenerate_channel_torrent(self, channel_pk, channel_id): self._logger.info("Regenerating personal channel %s %i", hexlify(channel_pk), channel_id) with db_session: channel = self.mds.ChannelMetadata.get(public_key=channel_pk, id_=channel_id) if channel is None: self._logger.warning( "Tried to regenerate non-existing channel %s %i", hexlify(channel_pk), channel_id) return None channel_dirname = channel.dirname for d in self.download_manager.get_downloads_by_name(channel_dirname): await self.download_manager.remove_download(d, remove_content=True) with db_session: channel = self.mds.ChannelMetadata.get_for_update( public_key=channel_pk, id_=channel_id) regenerated = channel.consolidate_channel_torrent() # If the user created their channel, but added no torrents to it, # the channel torrent will not be created. if regenerated is None: return None tdef = TorrentDef.load_from_dict(regenerated) self.updated_my_channel(tdef) return tdef
def test_tdef_init(): """ Test initializing a TorrentDef object """ tdef_params = TorrentDef( torrent_parameters={'announce': 'http://test.com'}) assert 'announce' in tdef_params.torrent_parameters
def check_watch_folder(self): if not self.watch_folder.is_dir(): return # Make sure that we pass a str to os.walk watch_dir = str(self.watch_folder) for root, _, files in os.walk(watch_dir): root = path_util.Path(root) for name in files: if not name.endswith(".torrent"): continue try: tdef = TorrentDef.load(root / name) if not tdef.get_metainfo(): self.cleanup_torrent_file(root, name) continue except: # torrent appears to be corrupt self.cleanup_torrent_file(root, name) continue infohash = tdef.get_infohash() if not self.download_manager.download_exists(infohash): self._logger.info("Starting download from torrent file %s", name) self.download_manager.start_download(torrent_file=root / name)
def test_restore_torrent_in_channel(metadata_store): """ Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it. """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) md = channel_metadata.add_torrent_to_channel(tdef, None) # Check correct re-add md.status = TODELETE md_updated = channel_metadata.add_torrent_to_channel(tdef, None) assert UPDATED == md.status assert md_updated == md assert md.has_valid_signature # Check update of torrent properties from a new tdef md.status = TODELETE new_tracker_address = 'http://tribler.org/announce' tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8') md_updated = channel_metadata.add_torrent_to_channel(tdef, None) assert md_updated == md assert md.status == UPDATED assert md.tracker_info == new_tracker_address assert md.has_valid_signature # In addition, check that the trackers table was properly updated assert len(md.health.trackers) == 2
def add_torrent(self, file, relative_path): _logger.info(f'Add torrent: {file}') directory = self.get_directory(relative_path) decoded_torrent = libtorrent.bdecode(file.read_bytes()) directory.add_torrent_to_channel(TorrentDef(metainfo=decoded_torrent), None)
def update_trackers(self, infohash, trackers): """ Update the trackers for a download. :param infohash: infohash of the torrent that needs to be updated :param trackers: A list of tracker urls. """ download = self.get_download(infohash) if download: old_def = download.get_def() old_trackers = old_def.get_trackers_as_single_tuple() new_trackers = list(set(trackers) - set(old_trackers)) all_trackers = list(old_trackers) + new_trackers if new_trackers: # Add new trackers to the download download.add_trackers(new_trackers) # Create a new TorrentDef if isinstance(old_def, TorrentDefNoMetainfo): new_def = TorrentDefNoMetainfo(old_def.get_infohash(), old_def.get_name(), download.get_magnet_link()) else: metainfo = old_def.get_metainfo() if len(all_trackers) > 1: metainfo["announce-list"] = [all_trackers] else: metainfo["announce"] = all_trackers[0] new_def = TorrentDef.load_from_dict(metainfo) # Set TorrentDef + checkpoint download.set_def(new_def) download.checkpoint()
def on_metadata_received_alert(self, _): torrent_info = get_info_from_handle(self.handle) if not torrent_info: return metadata = { b'info': bdecode_compat(torrent_info.metadata()), b'leechers': 0, b'seeders': 0 } trackers = [ tracker['url'].encode('utf-8') for tracker in self.handle.trackers() ] if len(trackers) > 1: metadata[b"announce-list"] = [trackers] elif trackers: metadata[b"announce"] = trackers[0] for peer in self.handle.get_peer_info(): if peer.progress == 1: metadata[b"seeders"] += 1 else: metadata[b"leechers"] += 1 try: self.tdef = TorrentDef.load_from_dict(metadata) except ValueError as ve: self._logger.exception(ve) return self.set_selected_files() self.checkpoint()
def add_torrents_from_dir(self, torrents_dir, recursive=False): torrents_list = [] errors_list = [] def rec_gen(dir_): for root, _, filenames in os.walk(dir_): for fn in filenames: yield Path(root) / fn filename_generator = rec_gen( torrents_dir) if recursive else os.listdir(torrents_dir) # Build list of .torrents to process torrents_list_generator = (Path(torrents_dir, f) for f in filename_generator) torrents_list = [ f for f in torrents_list_generator if f.is_file() and f.suffix == ".torrent" ] # 100 is a reasonable chunk size for commits for chunk in chunks(torrents_list, 100): for f in chunk: try: self.add_torrent_to_channel(TorrentDef.load(f)) except Exception: # pylint: disable=W0703 # Have to use the broad exception clause because Py3 versions of libtorrent # generate generic Exceptions errors_list.append(f) # Optimization to drop excess cache orm.commit() return torrents_list, errors_list
def personal_channel(metadata_store): global update_metainfo with db_session: chan = metadata_store.ChannelMetadata.create_channel( title="my test chan", description="test") tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) chan.add_torrent_to_channel(tdef, None) update_metainfo = chan.commit_channel_torrent() return chan
def load_checkpoint(self, filename): try: config = DownloadConfig.load(filename) except Exception: self._logger.exception("Could not open checkpoint file %s", filename) return metainfo = config.get_metainfo() if not metainfo: self._logger.error( "Could not resume checkpoint %s; metainfo not found", filename) return if not isinstance(metainfo, dict): self._logger.error( "Could not resume checkpoint %s; metainfo is not dict %s %s", filename, type(metainfo), repr(metainfo)) return try: url = metainfo.get(b'url', None) url = url.decode('utf-8') if url else url tdef = (TorrentDefNoMetainfo(metainfo[b'infohash'], metainfo[b'name'], url) if b'infohash' in metainfo else TorrentDef.load_from_dict(metainfo)) except (KeyError, ValueError) as e: self._logger.exception( "Could not restore tdef from metainfo dict: %s %s ", e, metainfo) return if config.get_bootstrap_download(): # In case the download is marked as bootstrap, remove it if its infohash does not # match the configured bootstrap infohash if hexlify(tdef.get_infohash()) != self.bootstrap_infohash: self.remove_config(tdef.get_infohash()) return config.state_dir = self.state_dir if config.get_dest_dir() == '': # removed torrent ignoring self._logger.info("Removing checkpoint %s destdir is %s", filename, config.get_dest_dir()) os.remove(filename) return try: if self.download_exists(tdef.get_infohash()): self._logger.info( "Not resuming checkpoint because download has already been added" ) else: self.start_download(tdef=tdef, config=config) except Exception: self._logger.exception( "Not resume checkpoint due to exception while adding download")
def test_add_torrent_to_channel(metadata_store): """ Test adding a torrent to your channel """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'}) assert channel_metadata.contents_list # Make sure trying to add a duplicate torrent does not result in an error channel_metadata.add_torrent_to_channel(tdef, None)
def gen_sample_channel(mds): my_channel = mds.ChannelMetadata.create_channel('test_channel', 'test description') my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_UBUNTU_FILE), None) my_channel.commit_channel_torrent() t2 = my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_VIDEO_FILE), None) mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) coll = mds.CollectionNode(origin_id=my_channel.id_, title='internal collection') mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) mds.TorrentMetadata.from_dict( dict(origin_id=coll.id_, **gen_random_entry())) my_channel.commit_channel_torrent() t2.soft_delete() my_channel.commit_channel_torrent() # Rename files to stable names mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob") torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent") os.rename(mdblob_name, CHANNEL_METADATA) os.rename(torrent_name, CHANNEL_TORRENT) # Update channel mds.TorrentMetadata.from_dict( dict(origin_id=my_channel.id_, **gen_random_entry())) my_channel.commit_channel_torrent() # Rename updated files to stable names os.rename(mdblob_name, CHANNEL_METADATA_UPDATED) os.rename(torrent_name, CHANNEL_TORRENT_UPDATED)
def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_disabled=False, hidden=False): self._logger.debug("Starting download: filename: %s, torrent def: %s", torrent_file, tdef) # the priority of the parameters is: (1) tdef, (2) torrent_file. # so if we have tdef, and torrent_file will be ignored, and so on. if tdef is None: if torrent_file is None: raise ValueError("Torrent file must be provided if tdef is not given") # try to get the torrent from the given torrent file tdef = TorrentDef.load(torrent_file) assert tdef is not None, "tdef MUST not be None after loading torrent" config = config or DownloadConfig() infohash = tdef.get_infohash() download = self.get_download(infohash) if download and infohash not in self.metainfo_requests: new_trackers = list(set(tdef.get_trackers_as_single_tuple()) - set(download.get_def().get_trackers_as_single_tuple())) if new_trackers: self.update_trackers(tdef.get_infohash(), new_trackers) return download # Create the destination directory if it does not exist yet try: if not config.get_dest_dir().is_dir(): os.makedirs(config.get_dest_dir()) except OSError: self._logger.error("Unable to create the download destination directory.") if config.get_time_added() == 0: config.set_time_added(int(timemod.time())) # Create the download download = Download(tdef=tdef, config=config, download_defaults=self.download_defaults, checkpoint_disabled=checkpoint_disabled, hidden=hidden or config.get_bootstrap_download(), notifier=self.notifier, state_dir=self.state_dir, download_manager=self, dummy=self.dummy_mode) atp = download.get_atp() # Keep metainfo downloads in self.downloads for now because we will need to remove it later, # and removing the download at this point will stop us from receiving any further alerts. if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download: self.downloads[infohash] = download if not self.dummy_mode: self.start_handle(download, atp) return download
async def test_get_metainfo_with_already_added_torrent(fake_dlmgr): """ Testing metainfo fetching for a torrent which is already in session. """ sample_torrent = TESTS_DATA_DIR / "bak_single.torrent" torrent_def = TorrentDef.load(sample_torrent) download_impl = MagicMock() download_impl.future_metainfo = succeed(bencode( torrent_def.get_metainfo())) download_impl.checkpoint = lambda: succeed(None) download_impl.stop = lambda: succeed(None) download_impl.shutdown = lambda: succeed(None) fake_dlmgr.initialize() fake_dlmgr.downloads[torrent_def.infohash] = download_impl assert await fake_dlmgr.get_metainfo(torrent_def.infohash)
def test_consolidate_channel_torrent(torrent_template, metadata_store): """ Test completely re-commit your channel """ channel = metadata_store.ChannelMetadata.create_channel('test', 'test') my_dir = Path(metadata_store.ChannelMetadata._channels_dir / channel.dirname).absolute() tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # 1st torrent torrent_entry = channel.add_torrent_to_channel(tdef, None) channel.commit_channel_torrent() # 2nd torrent metadata_store.TorrentMetadata.from_dict( dict(torrent_template, public_key=channel.public_key, origin_id=channel.id_, status=NEW)) channel.commit_channel_torrent() # Delete entry torrent_entry.soft_delete() channel.commit_channel_torrent() assert len(channel.contents_list) == 1 assert len(os.listdir(my_dir)) == 3 torrent3 = metadata_store.TorrentMetadata(public_key=channel.public_key, origin_id=channel.id_, status=NEW, infohash=random_infohash()) channel.commit_channel_torrent() torrent3.soft_delete() channel.consolidate_channel_torrent() assert len(os.listdir(my_dir)) == 1 metadata_store.TorrentMetadata.select( lambda g: g.metadata_type == REGULAR_TORRENT).delete() channel.local_version = 0 metadata_store.process_channel_dir(my_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) assert len(channel.contents[:]) == 1
async def start_download_from_uri(self, uri, config=None): scheme = scheme_from_uri(uri) if scheme in (HTTP_SCHEME, HTTPS_SCHEME): tdef = await TorrentDef.load_from_url(uri) return self.start_download(tdef=tdef, config=config) if scheme == MAGNET_SCHEME: name, infohash, _ = parse_magnetlink(uri) if infohash is None: raise RuntimeError("Missing infohash") if infohash in self.metainfo_cache: tdef = TorrentDef.load_from_dict(self.metainfo_cache[infohash]['meta_info']) else: tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri) return self.start_download(tdef=tdef, config=config) if scheme == FILE_SCHEME: file = uri_to_path(uri) return self.start_download(torrent_file=file, config=config) raise Exception("invalid uri")
async def test_add_torrent_duplicate(my_channel, rest_api): """ Test that adding a duplicate torrent to you channel does not result in an error """ with db_session: tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'}) with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file: base64_content = base64.b64encode( torrent_file.read()).decode('utf-8') post_params = {'torrent': base64_content} await do_request( rest_api, f'channels/{hexlify(my_channel.public_key)}/{my_channel.id_}/torrents', request_type='PUT', post_data=post_params, expected_code=200, )
def test_create_ffa_from_dict(metadata_store): """ Test creating a free-for-all torrent entry """ tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created signed_entry = metadata_store.TorrentMetadata.from_dict( tdef_to_metadata_dict(tdef)) metadata_store.TorrentMetadata.add_ffa_from_dict( tdef_to_metadata_dict(tdef)) assert metadata_store.TorrentMetadata.select( lambda g: g.public_key == EMPTY_BLOB).count() == 0 signed_entry.delete() # Create FFA entry metadata_store.TorrentMetadata.add_ffa_from_dict( tdef_to_metadata_dict(tdef)) assert metadata_store.TorrentMetadata.select( lambda g: g.public_key == EMPTY_BLOB).count() == 1
def test_delete_torrent_from_channel(metadata_store): """ Test deleting a torrent from your channel """ channel_metadata = metadata_store.ChannelMetadata.create_channel( 'test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) # Check that nothing is committed when deleting uncommited torrent metadata torrent = channel_metadata.add_torrent_to_channel(tdef, None) torrent.soft_delete() assert not channel_metadata.contents_list # Check append-only deletion process torrent = channel_metadata.add_torrent_to_channel(tdef, None) channel_metadata.commit_channel_torrent() assert len(channel_metadata.contents_list) == 1 torrent.soft_delete() channel_metadata.commit_channel_torrent() assert not channel_metadata.contents_list
def test_commit_channel_torrent(metadata_store): """ Test committing a channel torrent """ channel = metadata_store.ChannelMetadata.create_channel('test', 'test') tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) channel.add_torrent_to_channel(tdef, None) # The first run should return the infohash, the second should return None, because nothing was really done assert channel.commit_channel_torrent() assert not channel.commit_channel_torrent() # Test adding flags to channel torrent when adding thumbnail and description metadata_store.ChannelThumbnail(public_key=channel.public_key, origin_id=channel.id_, status=NEW) metadata_store.ChannelDescription(public_key=channel.public_key, origin_id=channel.id_, status=NEW) assert channel.commit_channel_torrent() assert channel.reserved_flags == 3 assert not channel.commit_channel_torrent()
async def start_download_from_uri(self, uri, config=None): if uri.startswith("http"): tdef = await TorrentDef.load_from_url(uri) return self.start_download(tdef=tdef, config=config) if uri.startswith("magnet:"): name, infohash, _ = parse_magnetlink(uri) if infohash is None: raise RuntimeError("Missing infohash") if infohash in self.metainfo_cache: tdef = TorrentDef.load_from_dict( self.metainfo_cache[infohash]['meta_info']) else: tdef = TorrentDefNoMetainfo( infohash, "Unknown name" if name is None else name, url=uri) return self.start_download(tdef=tdef, config=config) if uri.startswith("file:"): argument = uri_to_path(uri) return self.start_download(torrent_file=argument, config=config) raise Exception("invalid uri")
async def download_channel(self, channel): """ Download a channel with a given infohash and title. :param channel: The channel metadata ORM object. """ metainfo = await self.download_manager.get_metainfo(bytes( channel.infohash), timeout=60, hops=0) if metainfo is None: # Timeout looking for the channel metainfo. Probably, there are no seeds. # TODO: count the number of tries we had with the channel, so we can stop trying eventually return try: if metainfo[b'info'][b'name'].decode('utf-8') != channel.dirname: # Malformed channel # TODO: stop trying to download this channel until it is updated with a new infohash return except (KeyError, TypeError): return dcfg = DownloadConfig(state_dir=self.state_dir) dcfg.set_dest_dir(self.mds.channels_dir) dcfg.set_channel_download(True) tdef = TorrentDef(metainfo=metainfo) download = self.download_manager.start_download(tdef=tdef, config=dcfg, hidden=True) try: await download.future_finished except CancelledError: pass else: self.channels_processing_queue[channel.infohash] = ( PROCESS_CHANNEL_DIR, channel) return download
def test_updated_my_channel(personal_channel, gigachannel_manager, tmpdir): tdef = TorrentDef.load_from_dict(update_metainfo) gigachannel_manager.download_manager.start_download = Mock() gigachannel_manager.download_manager.download_exists = lambda *_: False gigachannel_manager.updated_my_channel(tdef) gigachannel_manager.download_manager.start_download.assert_called_once()
def channel_tdef(): return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' / 'channel_upd.torrent')