def on_save_resume_data_alert(self, alert): """ Callback for the alert that contains the resume data of a specific download. This resume data will be written to a file on disk. """ self._logger.debug(f'On save resume data alert: {alert}') if self.checkpoint_disabled: return resume_data = alert.resume_data # Make save_path relative if the torrent is saved in the Tribler state directory if self.state_dir and b'save_path' in resume_data: save_path = Path(resume_data[b'save_path'].decode('utf8')) if save_path.exists(): resume_data[b'save_path'] = str( save_path.normalize_to(self.state_dir)) metainfo = { 'infohash': self.tdef.get_infohash(), 'name': self.tdef.get_name_as_unicode(), 'url': self.tdef.get_url() } if isinstance(self.tdef, TorrentDefNoMetainfo) else self.tdef.get_metainfo() self.config.set_metainfo(metainfo) self.config.set_engineresumedata(resume_data) # Save it to file basename = hexlify(resume_data[b'info-hash']) + '.conf' filename = self.dlmgr.get_checkpoint_dir() / basename self.config.config['download_defaults'][ 'name'] = self.tdef.get_name_as_unicode( ) # store name (for debugging) self.config.write(str(filename)) self._logger.debug('Saving download config to file %s', filename)
async def test_regenerate_channel_torrent(self): with db_session: chan = self.generate_personal_channel() chan.commit_channel_torrent() chan_pk, chan_id = chan.public_key, chan.id_ channel_dir = Path(self.mock_session.mds.ChannelMetadata._channels_dir) / Path(chan.dirname) for f in channel_dir.iterdir(): f.unlink() # Test trying to regenerate a non-existing channel self.assertIsNone(await self.chanman.regenerate_channel_torrent(chan_pk, chan_id + 1)) # Mock existing downloads removal-related functions self.mock_session.dlmgr.get_downloads_by_name = lambda *_: [Mock()] downloads_to_remove = [] async def mock_remove_download(download_obj, **_): downloads_to_remove.append(download_obj) self.mock_session.dlmgr.remove_download = mock_remove_download # Test regenerating an empty channel self.mock_session.mds.ChannelMetadata.consolidate_channel_torrent = lambda *_: None self.assertIsNone(await self.chanman.regenerate_channel_torrent(chan_pk, chan_id)) self.assertEqual(1, len(downloads_to_remove)) # Test regenerating a non-empty channel self.chanman.updated_my_channel = Mock() self.mock_session.mds.ChannelMetadata.consolidate_channel_torrent = lambda *_: Mock() with patch("tribler_core.modules.libtorrent.torrentdef.TorrentDef.load_from_dict"): await self.chanman.regenerate_channel_torrent(chan_pk, chan_id) self.chanman.updated_my_channel.assert_called_once()
async def test_move_storage(self): """ Test that move_storage method works as expected by Libtorrent """ result = [] def mock_move(s): result.append(s) tdef = self.create_tdef() dl = Download(self.session, tdef) dl.setup() dl.handle = Mock() dl.handle.move_storage = mock_move dl.move_storage(Path("some_path")) self.assertEqual("some_path", result[0]) self.assertTrue("some_path", dl.config.get_dest_dir().name) await dl.shutdown() # Check the same thing, this time for TorrentDefNoMetainfo dl = Download(self.session, TorrentDefNoMetainfo(random_infohash(), "some_torrent")) dl.setup() dl.move_storage(Path("some_path")) self.assertEqual("some_path", result[0]) self.assertTrue("some_path", dl.config.get_dest_dir().name) await dl.shutdown()
def _existing_files(path_list): for path in path_list: path = Path(path) if not path.exists(): raise OSError(f'Path does not exist: {path}') elif path.is_file(): yield path
def test_skip_processing_of_received_personal_channel_torrents(metadata_store): """ Test that personal torrent is ignored by default when processing the torrent metadata payload """ channel = metadata_store.ChannelMetadata.create_channel('testchan') torrent_md = metadata_store.TorrentMetadata(origin_id=channel.id_, title='test', status=NEW, infohash=random_infohash()) channel.commit_channel_torrent() torrent_md.delete() channel_dir = Path( metadata_store.ChannelMetadata._channels_dir) / channel.dirname assert os.listdir(Path.fix_win_long_file(channel_dir)) # By default, personal channel torrent metadata processing is skipped so there should be no torrents # added to the channel channel.local_version = 0 metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_) assert not channel.contents # Enable processing of personal channel torrent metadata channel.local_version = 0 metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) assert len(channel.contents) == 1
def get_mdblob_sequence_number(filename): filepath = Path(filename) if filepath.suffixes == [BLOB_EXTENSION]: return int(filename.stem) if filepath.suffixes == [BLOB_EXTENSION, '.lz4']: return int(Path(filepath.stem).stem) return None
def get_base_path(): """ Get absolute path to resource, works for dev and for PyInstaller """ try: # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = Path(sys._MEIPASS) except Exception: base_path = Path(tribler_core.__file__).parent return Path(str_path(base_path))
def create_node(self, *args, **kwargs): metadata_store = MetadataStore( Path(self.temporary_directory()) / ("%d.db" % self.count), Path(self.temporary_directory()), default_eccrypto.generate_key(u"curve25519"), ) kwargs['metadata_store'] = metadata_store node = super(TestGigaChannelUnits, self).create_node(*args, **kwargs) self.count += 1 return node
def _get_path_to_requirements_txt() -> Optional[Path]: requirements_txt = 'requirements.txt' if scope == Scope.core: import tribler_core return Path(tribler_core.__file__).parent / requirements_txt if scope == Scope.gui: import tribler_gui return Path(tribler_gui.__file__).parent / requirements_txt raise AttributeError( f'Scope is {scope} but should be in {[s for s in Scope]}') # pylint: disable=unnecessary-comprehension
async def start_tribler(self, options): """ Main method to startup Tribler. """ async def signal_handler(sig): print("Received shut down signal %s" % sig) if not self._stopping: self._stopping = True await self.session.shutdown() print("Tribler shut down") get_event_loop().stop() self.process_checker.remove_lock_file() signal.signal(signal.SIGINT, lambda sig, _: ensure_future(signal_handler(sig))) signal.signal(signal.SIGTERM, lambda sig, _: ensure_future(signal_handler(sig))) statedir = Path(options.statedir or Path(get_appstate_dir(), '.Tribler')) config = TriblerConfig(statedir, config_file=statedir / 'triblerd.conf') # Check if we are already running a Tribler instance self.process_checker = ProcessChecker() if self.process_checker.already_running: print("Another Tribler instance is already using statedir %s" % config.get_state_dir()) get_event_loop().stop() return print("Starting Tribler") if options.restapi > 0: config.set_http_api_enabled(True) config.set_http_api_port(options.restapi) if options.ipv8 > 0: config.set_ipv8_port(options.ipv8) elif options.ipv8 == 0: config.set_ipv8_enabled(False) if options.libtorrent != -1 and options.libtorrent > 0: config.set_libtorrent_port(options.libtorrent) if options.ipv8_bootstrap_override is not None: config.set_ipv8_bootstrap_override(options.ipv8_bootstrap_override) if options.testnet: config.set_testnet(True) self.session = Session(config) try: await self.session.start() except Exception as e: print(str(e)) get_event_loop().stop() else: print("Tribler started")
def create_node(self, *args, **kwargs): metadata_store = MetadataStore( Path(self.temporary_directory()) / f"{self.count}.db", Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519"), disable_sync=True, ) kwargs['metadata_store'] = metadata_store node = super().create_node(*args, **kwargs) self.count += 1 return node
def create_node(self, *args, **kwargs): metadata_store = MetadataStore( Path(self.temporary_directory()) / f"{self.count}.db", Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519"), disable_sync=True, ) kwargs['metadata_store'] = metadata_store with mock.patch('tribler_core.modules.metadata_store.community.gigachannel_community.DiscoveryBooster'): node = super().create_node(*args, **kwargs) self.count += 1 return node
async def test_get_path_relative(tmpdir): config = TriblerConfig(state_dir=tmpdir) config.general.log_dir = None assert not config.general.log_dir config.general.log_dir = '.' assert config.general.get_path_as_absolute('log_dir', tmpdir) == Path(tmpdir) config.general.log_dir = '1' assert config.general.get_path_as_absolute('log_dir', tmpdir) == Path(tmpdir) / '1'
def create_node(self, *args, **kwargs): metadata_store = MetadataStore( Path(self.temporary_directory()) / f"{self.count}.db", Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519"), disable_sync=True, ) self.metadata_store_set.add(metadata_store) kwargs['metadata_store'] = metadata_store kwargs['rqc_settings'] = RemoteQueryCommunitySettings() node = super().create_node(*args, **kwargs) self.count += 1 return node
def create_node(self, *args, **kwargs): mds = MetadataStore( Path(self.temporary_directory()) / ("%d.db" % self.count), Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519")) torrent_checker = MockObject() torrent_checker.torrents_checked = set() return MockIPv8("curve25519", PopularityCommunity, metadata_store=mds, torrent_checker=torrent_checker)
def get_dest_dir(self): """ Gets the directory where to save this Download. """ dest_dir = self.config['download_defaults']['saveas'] if not dest_dir: dest_dir = get_default_dest_dir() self.set_dest_dir(dest_dir) # This is required to support relative paths if not Path(dest_dir).is_absolute(): dest_dir = self.state_dir / dest_dir return Path(dest_dir)
def get_default_dest_dir(): """ Returns the default dir to save content to. """ tribler_downloads = Path("TriblerDownloads") if tribler_downloads.is_dir(): return tribler_downloads.resolve() home = get_home_dir() downloads = home / "Downloads" if downloads.is_dir(): return (downloads / tribler_downloads).resolve() return (home / tribler_downloads).resolve()
def create_node(self, *args, **kwargs): self.metadata_store = MetadataStore( Path(self.temporary_directory()) / "mds.db", Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519"), disable_sync=True, ) self.tags_db = TagDatabase( str(Path(self.temporary_directory()) / "tags.db")) kwargs['metadata_store'] = self.metadata_store kwargs['tags_db'] = self.tags_db kwargs['rqc_settings'] = RemoteQueryCommunitySettings() return super().create_node(*args, **kwargs)
def create_node(self, *args, **kwargs): metadata_store = MetadataStore( Path(self.temporary_directory()) / f"{self.count}.db", Path(self.temporary_directory()), default_eccrypto.generate_key("curve25519"), disable_sync=True, ) self.metadata_store_set.add(metadata_store) kwargs['metadata_store'] = metadata_store kwargs['settings'] = ChantSettings() kwargs['rqc_settings'] = RemoteQueryCommunitySettings() with mock.patch('tribler_core.components.gigachannel.community.gigachannel_community.DiscoveryBooster'): node = super().create_node(*args, **kwargs) self.count += 1 return node
async def test_remove_with_files(self): """ Testing whether the API returns 200 if a download is being removed """ # Create a copy of the file, so we can remove it later source_file = TESTS_DATA_DIR / 'video.avi' tmpdir = self.temporary_directory() copied_file = tmpdir / Path(source_file).name shutil.copyfile(source_file, copied_file) video_tdef, _ = self.create_local_torrent(copied_file) dcfg = DownloadConfig() dcfg.set_dest_dir(tmpdir) download = self.session.dlmgr.start_download(tdef=video_tdef, config=dcfg) infohash = get_hex_infohash(video_tdef) while not download.handle: await sleep(0.1) await sleep(2) await self.do_request('downloads/%s' % infohash, post_data={"remove_data": True}, expected_code=200, request_type='DELETE', expected_json={ u"removed": True, u"infohash": u"c9a19e7fe5d9a6c106d6ea3c01746ac88ca3c7a5" }) while copied_file.exists(): await sleep(0.1) self.assertEqual(len(self.session.dlmgr.get_downloads()), 0) self.assertFalse(copied_file.exists())
def test_multiple_squashed_commit_and_read(metadata_store): """ Test committing entries into several squashed blobs and reading them back """ metadata_store.ChannelMetadata._CHUNK_SIZE_LIMIT = 500 num_entries = 10 channel = metadata_store.ChannelMetadata.create_channel('testchan') md_list = [ metadata_store.TorrentMetadata(origin_id=channel.id_, title='test' + str(x), status=NEW, infohash=random_infohash()) for x in range(0, num_entries) ] channel.commit_channel_torrent() channel.local_version = 0 for md in md_list: md.delete() channel_dir = Path( metadata_store.ChannelMetadata._channels_dir) / channel.dirname assert len(os.listdir(channel_dir) ) > 1 # make sure it was broken into more than one .mdblob file metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) assert num_entries == len(channel.contents)
def set_dest_dir(self, path): """ Sets the directory where to save this Download. @param path A path of a directory. """ # If something is saved inside the Tribler state dir, it should use relative path path = Path(path).normalize_to(self.state_dir) self.config['download_defaults']['saveas'] = str(path)
def get_files_info_json(self, download): """ Return file information as JSON from a specified download. """ files_json = [] files_completion = dict( (name, progress) for name, progress in download.get_state().get_files_completion()) selected_files = download.config.get_selected_files() file_index = 0 for fn, size in download.get_def().get_files_with_length(): files_json.append({ "index": file_index, "name": str(Path(fn)), "size": size, "included": (file_index in selected_files or not selected_files), "progress": files_completion.get(fn, 0.0) }) file_index += 1 return files_json
def test_relative_paths(self): # Default should be taken from config.spec self.assertEqual( self.tribler_config.get_trustchain_keypair_filename(), path_util.abspath(self.state_dir / "ec_multichain.pem")) local_name = Path("somedir") / "ec_multichain.pem" global_name = self.state_dir / local_name self.tribler_config.set_trustchain_keypair_filename(global_name) # It should always return global path self.assertEqual(self.tribler_config.get_trustchain_keypair_filename(), global_name) # But internally it should be stored as a local path string self.assertEqual( self.tribler_config.config['trustchain']['ec_keypair_filename'], str(local_name)) # If it points out of the state dir, it should be saved as a global path string out_of_dir_name_global = path_util.abspath(self.state_dir / ".." / "filename").resolve() self.tribler_config.set_trustchain_keypair_filename( out_of_dir_name_global) self.assertEqual( self.tribler_config.config['trustchain']['ec_keypair_filename'], str(out_of_dir_name_global))
def initialize(self): # Create the checkpoints directory (self.state_dir / STATEDIR_CHECKPOINT_DIR).mkdir(exist_ok=True) # Start upnp if self.config.upnp: self.get_session().start_upnp() if has_bep33_support() and self.download_defaults.number_hops <= len(self.socks_listen_ports or []): # Also listen to DHT log notifications - we need the dht_pkt_alert and extract the BEP33 bloom filters dht_health_session = self.get_session(self.download_defaults.number_hops) dht_health_session.set_alert_mask(self.default_alert_mask | lt.alert.category_t.dht_log_notification) self.dht_health_manager = DHTHealthManager(dht_health_session) # Make temporary directory for metadata collecting through DHT self.metadata_tmpdir = self.metadata_tmpdir or Path.mkdtemp(suffix='tribler_metainfo_tmpdir') # Register tasks self.register_task("process_alerts", self._task_process_alerts, interval=1) if self.dht_readiness_timeout > 0 and self.config.dht: self._dht_ready_task = self.register_task("check_dht_ready", self._check_dht_ready) self.register_task("request_torrent_updates", self._request_torrent_updates, interval=1) self.register_task('task_cleanup_metacache', self._task_cleanup_metainfo_cache, interval=60, delay=0) self.set_download_states_callback(self.sesscb_states_callback)
def create_node(self): config = TunnelCommunitySettings() mock_ipv8 = MockIPv8("curve25519", TriblerTunnelCommunity, settings={'remove_tunnel_delay': 0}, config=config, exitnode_cache=Path(self.temporary_directory()) / "exitnode_cache.dat") mock_ipv8.overlay.settings.max_circuits = 1 db = BandwidthDatabase( db_path=MEMORY_DB, my_pub_key=mock_ipv8.my_peer.public_key.key_to_bin()) # Load the bandwidth accounting community mock_ipv8.overlay.bandwidth_community = BandwidthAccountingCommunity( mock_ipv8.my_peer, mock_ipv8.endpoint, mock_ipv8.network, settings=BandwidthAccountingSettings(), database=db) mock_ipv8.overlay.dht_provider = MockDHTProvider( Peer(mock_ipv8.overlay.my_peer.key, mock_ipv8.overlay.my_estimated_wan)) return mock_ipv8
def test_get_files_with_length(self): name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86' name_unicode = name_bytes.decode() t = TorrentDef() t.metainfo = { b'info': { b'files': [{ b'path.utf-8': [name_bytes], b'length': 123 }, { b'path.utf-8': [b'file.txt'], b'length': 456 }] } } self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123), (Path('file.txt'), 456)]) t.metainfo = { b'info': { b'files': [{ b'path': [name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123), (Path('file.txt'), 456)]) t.metainfo = { b'info': { b'files': [{ b'path': [b'test\xff' + name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } self.assertEqual(t.get_files_with_length(), [(Path('test?????????????'), 123), (Path('file.txt'), 456)]) t.metainfo = { b'info': { b'files': [{ b'path.utf-8': [b'test\xff' + name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } self.assertEqual(t.get_files_with_length(), [(Path('file.txt'), 456)])
def consolidate_channel_torrent(self): """ Delete the channel dir contents and create it anew. Use it to consolidate fragmented channel torrent directories. :param key: The public/private key, used to sign the data """ # Remark: there should be a way to optimize this stuff with SQL and better tree traversal algorithms # Cleanup entries marked for deletion db.CollectionNode.collapse_deleted_subtrees() # Note: It should be possible to stop alling get_contents_to_commit here commit_queue = self.get_contents_to_commit() for entry in commit_queue: if entry.status == TODELETE: entry.delete() folder = Path(self._channels_dir) / self.dirname # We check if we need to re-create the channel dir in case it was deleted for some reason if not folder.is_dir(): os.makedirs(folder) for filename in os.listdir(folder): file_path = folder / filename # We only remove mdblobs and leave the rest as it is if filename.endswith(BLOB_EXTENSION) or filename.endswith( BLOB_EXTENSION + '.lz4'): os.unlink(Path.fix_win_long_file(file_path)) # Channel should get a new starting timestamp and its contents should get higher timestamps start_timestamp = clock.tick() def update_timestamps_recursive(node): if issubclass(type(node), db.CollectionNode): for child in node.contents: update_timestamps_recursive(child) if node.status in [COMMITTED, UPDATED, NEW]: node.status = UPDATED node.timestamp = clock.tick() node.sign() update_timestamps_recursive(self) return self.commit_channel_torrent( new_start_timestamp=start_timestamp)
async def start(self, options): # Determine ipv8 port ipv8_port = options.ipv8_port if ipv8_port == -1 and "HELPER_INDEX" in os.environ and "HELPER_BASE" in os.environ: base_port = int(os.environ["HELPER_BASE"]) ipv8_port = base_port + int(os.environ["HELPER_INDEX"]) * 5 statedir = Path( os.path.join(get_root_state_directory(), "tunnel-%d") % ipv8_port) config = TriblerConfig(statedir, config_file=statedir / 'triblerd.conf') config.set_tunnel_community_socks5_listen_ports([]) config.set_tunnel_community_random_slots(options.random_slots) config.set_tunnel_community_competing_slots(options.competing_slots) config.set_torrent_checking_enabled(False) config.set_ipv8_enabled(True) config.set_libtorrent_enabled(False) config.set_ipv8_port(ipv8_port) config.set_ipv8_address(options.ipv8_address) config.set_trustchain_enabled(True) config.set_market_community_enabled(False) config.set_dht_enabled(True) config.set_tunnel_community_exitnode_enabled(bool(options.exit)) config.set_popularity_community_enabled(False) config.set_testnet(bool(options.testnet)) config.set_chant_enabled(False) config.set_bootstrap_enabled(False) if not options.no_rest_api: config.set_http_api_enabled(True) api_port = options.restapi if "HELPER_INDEX" in os.environ and "HELPER_BASE" in os.environ: api_port = int(os.environ["HELPER_BASE"]) + 10000 + int( os.environ["HELPER_INDEX"]) config.set_http_api_port(api_port) if options.ipv8_bootstrap_override is not None: config.set_ipv8_bootstrap_override(options.ipv8_bootstrap_override) self.session = Session(config) self.log_circuits = options.log_circuits self.session.notifier.add_observer(NTFY.TUNNEL_REMOVE, self.circuit_removed) await self.session.start() if options.log_rejects: # We set this after Tribler has started since the tunnel_community won't be available otherwise self.session.tunnel_community.reject_callback = self.on_circuit_reject # Tunnel helpers store more TrustChain blocks self.session.trustchain_community.settings.max_db_blocks = 5000000 self.tribler_started()
def test_get_files_with_length(tdef): name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86' name_unicode = name_bytes.decode() tdef.metainfo = { b'info': { b'files': [{ b'path.utf-8': [name_bytes], b'length': 123 }, { b'path.utf-8': [b'file.txt'], b'length': 456 }] } } assert tdef.get_files_with_length() == [(Path(name_unicode), 123), (Path('file.txt'), 456)] tdef.metainfo = { b'info': { b'files': [{ b'path': [name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } assert tdef.get_files_with_length() == [(Path(name_unicode), 123), (Path('file.txt'), 456)] tdef.metainfo = { b'info': { b'files': [{ b'path': [b'test\xff' + name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } assert tdef.get_files_with_length() == [(Path('test?????????????'), 123), (Path('file.txt'), 456)] tdef.metainfo = { b'info': { b'files': [{ b'path.utf-8': [b'test\xff' + name_bytes], b'length': 123 }, { b'path': [b'file.txt'], b'length': 456 }] } } assert tdef.get_files_with_length() == [(Path('file.txt'), 456)]