def test_open_db_script_file(self): sqlite_test_2 = SQLiteCacheDB(os.path.join(self.session_base_dir, "test_db.db"), DB_SCRIPT_ABSOLUTE_PATH) sqlite_test_2.initialize() sqlite_test_2.initial_begin() sqlite_test_2.write_version(4) self.assertEqual(sqlite_test_2.version, 4)
def test_clean_db(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), self.tribler_db_script) sqlite_test_2.initialize() sqlite_test_2.initial_begin() sqlite_test_2.clean_db(vacuum=True, exiting=True)
class AbstractUpgrader(TriblerCoreTest): FILE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) DATABASES_DIR = os.path.abspath( os.path.join(FILE_DIR, u"../data/upgrade_databases/")) def write_data_to_file(self, file_name): with open(file_name, 'w') as file: file.write("lorem ipsum") file.close() @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self): yield super(AbstractUpgrader, self).setUp() self.config = TriblerConfig(ConfigObj(configspec=CONFIG_SPEC_PATH)) self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_collecting_dir( os.path.join(self.session_base_dir, 'torrent_collecting_dir')) self.session = Session(self.config) self.sqlitedb = None self.torrent_store = None def tearDown(self): if self.torrent_store: self.torrent_store.close() super(AbstractUpgrader, self).tearDown() if self.sqlitedb: self.sqlitedb.close() self.sqlitedb = None self.session = None def copy_and_initialize_upgrade_database(self, db_name): # create a file to be removed in the thumbnails os.mkdir(self.session.config.get_torrent_collecting_dir()) os.mkdir( os.path.join(self.session.config.get_torrent_collecting_dir(), 'dir1')) self.write_data_to_file( os.path.join(self.session.config.get_torrent_collecting_dir(), 'dir1', 'file1.txt')) os.mkdir(os.path.join(self.session_base_dir, 'sqlite')) shutil.copyfile( os.path.join(self.DATABASES_DIR, db_name), os.path.join(self.session.config.get_state_dir(), 'sqlite', 'tribler.sdb')) shutil.copyfile( os.path.join(self.DATABASES_DIR, 'torrent_upgrade_64_dispersy.db'), os.path.join(self.session.config.get_state_dir(), 'sqlite', 'dispersy.db')) db_path = os.path.join(self.session.config.get_state_dir(), 'sqlite', 'tribler.sdb') self.sqlitedb = SQLiteCacheDB(db_path) self.sqlitedb.initialize() self.sqlitedb.initial_begin() self.session.sqlite_db = self.sqlitedb
def test_clean_db(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), DB_SCRIPT_ABSOLUTE_PATH) sqlite_test_2.initialize() sqlite_test_2.initial_begin() sqlite_test_2.clean_db(vacuum=True, exiting=True)
class AbstractDB(AbstractServer): def setUp(self): super(AbstractDB, self).setUp() # dummy session self.config = SessionStartupConfig() self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking(False) self.config.set_multicast_local_peer_discovery(False) self.config.set_megacache(False) self.config.set_dispersy(False) self.config.set_mainline_dht(False) self.config.set_torrent_collecting(False) self.config.set_libtorrent(False) self.config.set_dht_torrent_collecting(False) self.config.set_videoplayer(False) self.config.set_torrent_store(False) self.session = Session(self.config, ignore_singleton=True) dbpath = init_bak_tribler_sdb('bak_new_tribler.sdb', destination_path=self.getStateDir(), overwrite=True) self.sqlitedb = SQLiteCacheDB(self.session, busytimeout=BUSYTIMEOUT) self.sqlitedb.initialize(dbpath) self.session.sqlite_db = self.sqlitedb @blocking_call_on_reactor_thread def tearDown(self): self.sqlitedb.close() self.sqlitedb = None self.session.del_instance() self.session = None super(AbstractDB, self).tearDown(self)
def test_no_permission_on_script(self): db_path = os.path.join(self.session_base_dir, "test_db.db") new_script_path = os.path.join(self.session_base_dir, "script.sql") shutil.copyfile(self.tribler_db_script, new_script_path) os.chmod(new_script_path, 0) sqlite_test_2 = SQLiteCacheDB(db_path, new_script_path) sqlite_test_2.initialize()
def test_integrity_check_failed(self): sqlite_test_2 = SQLiteCacheDB(os.path.join(self.session_base_dir, "test_db.db"), os.path.join(self.SQLITE_SCRIPTS_DIR, "script1.sql")) def execute(sql): if sql == u"PRAGMA quick_check": db_response = MockObject() db_response.next = lambda: ("Error: database disk image is malformed", ) return db_response sqlite_test_2.execute = execute sqlite_test_2.initialize()
class TestSqliteCacheDB(AbstractServer): def setUp(self): super(TestSqliteCacheDB, self).setUp() self.config = SessionStartupConfig() self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking(False) self.config.set_multicast_local_peer_discovery(False) self.config.set_megacache(False) self.config.set_dispersy(False) self.config.set_mainline_dht(False) self.config.set_torrent_collecting(False) self.config.set_libtorrent(False) self.config.set_dht_torrent_collecting(False) self.config.set_videoplayer(False) self.session = Session(self.config, ignore_singleton=True) self.sqlitedb = None def tearDown(self): super(TestSqliteCacheDB, self).tearDown() if self.sqlitedb: self.sqlitedb.close() self.sqlitedb = None self.session.del_instance() self.session = None def test_upgrade_from_obsolete_version(self): """We no longer support DB versions older than 17 (Tribler 6.0)""" dbpath = init_bak_tribler_sdb(u"bak_old_tribler.sdb", destination_path=self.getStateDir(), overwrite=True) self.sqlitedb = SQLiteCacheDB(self.session) self.sqlitedb.initialize(dbpath) class MockTorrentStore(object): def flush(): pass def close(): pass db_migrator = DBUpgrader(self.session, self.sqlitedb, torrent_store=MockTorrentStore()) self.assertRaises(VersionNoLongerSupportedError, db_migrator.start_migrate) def test_upgrade_from_17(self): pass
class AbstractDB(TriblerCoreTest): def setUpPreSession(self): self.config = SessionStartupConfig() self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking(False) self.config.set_multicast_local_peer_discovery(False) self.config.set_megacache(False) self.config.set_dispersy(False) self.config.set_mainline_dht(False) self.config.set_torrent_collecting(False) self.config.set_libtorrent(False) self.config.set_dht_torrent_collecting(False) self.config.set_videoserver_enabled(False) self.config.set_torrent_store(False) @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self): yield super(AbstractDB, self).setUp() self.setUpPreSession() self.session = Session(self.config, ignore_singleton=True) tar = tarfile.open( os.path.join(TESTS_DATA_DIR, 'bak_new_tribler.sdb.tar.gz'), 'r|gz') tar.extractall(self.session_base_dir) db_path = os.path.join(self.session_base_dir, 'bak_new_tribler.sdb') db_script_path = os.path.join(get_lib_path(), DB_SCRIPT_NAME) self.sqlitedb = SQLiteCacheDB(db_path, db_script_path, busytimeout=BUSYTIMEOUT) self.sqlitedb.initialize() self.session.sqlite_db = self.sqlitedb def tearDown(self): self.sqlitedb.close() self.sqlitedb = None self.session.del_instance() self.session = None super(AbstractDB, self).tearDown(self)
def test_integrity_check_triggered(self): """ Tests if integrity check is triggered if temporary rollback files are present.""" def do_integrity_check(): do_integrity_check.called = True db_path = os.path.join(self.session_base_dir, "test_db.db") sqlite_test = SQLiteCacheDB(db_path) sqlite_test.do_quick_integrity_check = do_integrity_check do_integrity_check.called = False sqlite_test.initialize() self.assertFalse(do_integrity_check.called) db_path2 = os.path.join(self.session_base_dir, "test_db2.db") wal_file = open(os.path.join(self.session_base_dir, "test_db2.db-shm"), 'w') wal_file.close() sqlite_test_2 = SQLiteCacheDB(db_path2) sqlite_test_2.do_quick_integrity_check = do_integrity_check do_integrity_check.called = False sqlite_test_2.initialize() self.assertTrue(do_integrity_check.called)
class AbstractDB(TriblerCoreTest): def setUpPreSession(self): self.config = TriblerConfig(ConfigObj(configspec=CONFIG_SPEC_PATH)) self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking_enabled(False) self.config.set_megacache_enabled(False) self.config.set_dispersy_enabled(False) self.config.set_mainline_dht_enabled(False) self.config.set_torrent_collecting_enabled(False) self.config.set_libtorrent_enabled(False) self.config.set_video_server_enabled(False) self.config.set_torrent_store_enabled(False) @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self): yield super(AbstractDB, self).setUp() self.setUpPreSession() self.session = Session(self.config, ignore_singleton=True) tar = tarfile.open( os.path.join(TESTS_DATA_DIR, 'bak_new_tribler.sdb.tar.gz'), 'r|gz') tar.extractall(self.session_base_dir) db_path = os.path.join(self.session_base_dir, 'bak_new_tribler.sdb') self.sqlitedb = SQLiteCacheDB(db_path, busytimeout=BUSYTIMEOUT) self.sqlitedb.initialize() self.session.sqlite_db = self.sqlitedb def tearDown(self): self.sqlitedb.close() self.sqlitedb = None self.session.del_instance() self.session = None super(AbstractDB, self).tearDown(self)
def test_failed_commit(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), DB_SCRIPT_ABSOLUTE_PATH) sqlite_test_2.initialize() sqlite_test_2.write_version(4)
def test_open_db_script_file_directory(self): file_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), file_dir) sqlite_test_2.initialize()
class TestSqliteCacheDB(TriblerCoreTest): FILE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) SQLITE_SCRIPTS_DIR = os.path.abspath( os.path.join(FILE_DIR, u"data/sqlite_scripts/")) @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self): yield super(TestSqliteCacheDB, self).setUp() db_path = u":memory:" self.sqlite_test = SQLiteCacheDB(db_path) self.sqlite_test.set_show_sql(True) self.sqlite_test.initialize() import Tribler self.tribler_db_script = os.path.join( os.path.dirname(Tribler.__file__), DB_SCRIPT_NAME) def tearDown(self): super(TestSqliteCacheDB, self).tearDown() self.sqlite_test.close() self.sqlite_test = None @blocking_call_on_reactor_thread def test_create_db(self): sql = u"CREATE TABLE person(lastname, firstname);" self.sqlite_test.execute(sql) self.assertIsInstance(self.sqlite_test.version, int) @blocking_call_on_reactor_thread @raises(OSError) def test_no_file_db_error(self): file_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) sqlite_test_2 = SQLiteCacheDB(file_dir) sqlite_test_2.initialize() @blocking_call_on_reactor_thread def test_open_db_new_file(self): db_path = os.path.join(self.session_base_dir, "test_db.db") sqlite_test_2 = SQLiteCacheDB(db_path) sqlite_test_2.initialize() self.assertTrue(os.path.isfile(db_path)) @blocking_call_on_reactor_thread @raises(OSError) def test_open_db_script_file_invalid_location(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), u'myfakelocation') sqlite_test_2.initialize() @blocking_call_on_reactor_thread @raises(OSError) def test_open_db_script_file_directory(self): file_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), file_dir) sqlite_test_2.initialize() @blocking_call_on_reactor_thread def test_open_db_script_file(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), self.tribler_db_script) sqlite_test_2.initialize() sqlite_test_2.initial_begin() sqlite_test_2.write_version(4) self.assertEqual(sqlite_test_2.version, 4) @blocking_call_on_reactor_thread def test_initial_begin(self): self.sqlite_test.initial_begin() @blocking_call_on_reactor_thread @raises(SQLError) def test_failed_commit(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), self.tribler_db_script) sqlite_test_2.initialize() sqlite_test_2.write_version(4) @blocking_call_on_reactor_thread @raises(Exception) def test_initial_begin_no_connection(self): db_path = os.path.join(self.session_base_dir, "test_db.db") sqlite_test_2 = SQLiteCacheDB(db_path) sqlite_test_2.initial_begin() @blocking_call_on_reactor_thread @skipIf(sys.platform == "win32", "chmod does not work on Windows") @raises(IOError) def test_no_permission_on_script(self): db_path = os.path.join(self.session_base_dir, "test_db.db") new_script_path = os.path.join(self.session_base_dir, "script.sql") shutil.copyfile(self.tribler_db_script, new_script_path) os.chmod(new_script_path, 0) sqlite_test_2 = SQLiteCacheDB(db_path, new_script_path) sqlite_test_2.initialize() @blocking_call_on_reactor_thread @raises(CorruptedDatabaseError) def test_no_version_info_in_database(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), os.path.join(self.SQLITE_SCRIPTS_DIR, "script1.sql")) sqlite_test_2.initialize() @blocking_call_on_reactor_thread @raises(CorruptedDatabaseError) def test_integrity_check_failed(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), os.path.join(self.SQLITE_SCRIPTS_DIR, "script1.sql")) def execute(sql): if sql == u"PRAGMA quick_check": db_response = MockObject() db_response.next = lambda: ( "Error: database disk image is malformed", ) return db_response sqlite_test_2.execute = execute sqlite_test_2.initialize() @blocking_call_on_reactor_thread def test_clean_db(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), self.tribler_db_script) sqlite_test_2.initialize() sqlite_test_2.initial_begin() sqlite_test_2.clean_db(vacuum=True, exiting=True) @blocking_call_on_reactor_thread @skipIf(sys.platform == "win32", "chmod does not work on Windows") @raises(CantOpenError) def test_open_db_connection_no_permission(self): os.chmod(os.path.join(self.session_base_dir), 0) sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db")) sqlite_test_2.initialize() @blocking_call_on_reactor_thread def test_insert(self): self.test_create_db() self.sqlite_test.insert('person', lastname='a', firstname='b') self.assertEqual(self.sqlite_test.size('person'), 1) @blocking_call_on_reactor_thread def test_fetchone(self): self.test_insert() one = self.sqlite_test.fetchone(u"SELECT * FROM person") self.assertEqual(one, ('a', 'b')) one = self.sqlite_test.fetchone( u"SELECT lastname FROM person WHERE firstname == 'b'") self.assertEqual(one, 'a') one = self.sqlite_test.fetchone( u"SELECT lastname FROM person WHERE firstname == 'c'") self.assertIsNone(one) @blocking_call_on_reactor_thread def test_insertmany(self): self.test_create_db() values = [] for i in range(100): value = (str(i), str(i**2)) values.append(value) self.sqlite_test.insertMany('person', values) self.assertEqual(self.sqlite_test.size('person'), 100) @blocking_call_on_reactor_thread def test_fetchall(self): self.test_insertmany() all = self.sqlite_test.fetchall('select * from person') self.assertEqual(len(all), 100) all = self.sqlite_test.fetchall( "select * from person where lastname=='101'") self.assertEqual(all, []) @blocking_call_on_reactor_thread def test_insertorder(self): self.test_insertmany() self.sqlite_test.insert('person', lastname='1', firstname='abc') one = self.sqlite_test.fetchone( "select firstname from person where lastname == '1'") self.assertTrue(one == '1' or one == 'abc') all = self.sqlite_test.fetchall( "select firstname from person where lastname == '1'") self.assertEqual(len(all), 2) @blocking_call_on_reactor_thread def test_update(self): self.test_insertmany() self.sqlite_test.update('person', "lastname == '2'", firstname='56') one = self.sqlite_test.fetchone( "select firstname from person where lastname == '2'") self.assertEqual(one, '56') self.sqlite_test.update('person', "lastname == '3'", firstname=65) one = self.sqlite_test.fetchone( "select firstname from person where lastname == '3'") self.assertEqual(one, 65) self.sqlite_test.update('person', "lastname == '4'", firstname=654, lastname=44) one = self.sqlite_test.fetchone( "select firstname from person where lastname == 44") self.assertEqual(one, 654) @blocking_call_on_reactor_thread def test_delete_single_element(self): """ This test tests whether deleting using a single element as value works. """ self.test_insert() self.sqlite_test.insert('person', lastname='x', firstname='z') one = self.sqlite_test.fetchone(u"SELECT * FROM person") self.assertEqual(one, ('a', 'b')) self.sqlite_test.delete("person", lastname="a") one = self.sqlite_test.fetchone(u"SELECT * FROM person") self.assertEqual(one, ('x', 'z')) @blocking_call_on_reactor_thread def test_delete_tuple(self): """ This test tests whether deleting using a tuple as value works. """ self.test_insert() self.sqlite_test.insert('person', lastname='x', firstname='z') one = self.sqlite_test.fetchone(u"SELECT * FROM person") self.assertEqual(one, ('a', 'b')) self.sqlite_test.delete("person", lastname=("LIKE", "a")) one = self.sqlite_test.fetchone(u"SELECT * FROM person") self.assertEqual(one, ('x', 'z'))
class Session(SessionConfigInterface): """ A Session is a running instance of the Tribler Core and the Core's central class. It implements the SessionConfigInterface which can be used to change session parameters at runtime (for selected parameters). cf. libtorrent session """ __single = None def __init__(self, scfg=None, ignore_singleton=False, autoload_discovery=True): """ A Session object is created which is configured following a copy of the SessionStartupConfig scfg. (copy constructor used internally) @param scfg SessionStartupConfig object or None, in which case we look for a saved session in the default location (state dir). If we can't find it, we create a new SessionStartupConfig() object to serve as startup config. Next, the config is saved in the directory indicated by its 'state_dir' attribute. In the current implementation only a single session instance can exist at a time in a process. The ignore_singleton flag is used for testing. """ if not ignore_singleton: if Session.__single: raise RuntimeError("Session is singleton") Session.__single = self self._logger = logging.getLogger(self.__class__.__name__) self.ignore_singleton = ignore_singleton self.sesslock = NoDispersyRLock() # Determine startup config to use if scfg is None: # If no override scfg = SessionStartupConfig.load() else: # overrides any saved config # Work from copy scfg = SessionStartupConfig(copy.copy(scfg.sessconfig)) def create_dir(fullpath): if not os.path.isdir(fullpath): os.makedirs(fullpath) def set_and_create_dir(dirname, setter, default_dir): if dirname is None: setter(default_dir) create_dir(dirname or default_dir) state_dir = scfg.get_state_dir() set_and_create_dir(state_dir, scfg.set_state_dir, state_dir) set_and_create_dir(scfg.get_torrent_store_dir(), scfg.set_torrent_store_dir, os.path.join(scfg.get_state_dir(), STATEDIR_TORRENT_STORE_DIR)) # metadata store set_and_create_dir(scfg.get_metadata_store_dir(), scfg.set_metadata_store_dir, os.path.join(scfg.get_state_dir(), STATEDIR_METADATA_STORE_DIR)) set_and_create_dir(scfg.get_peer_icon_path(), scfg.set_peer_icon_path, os.path.join(scfg.get_state_dir(), STATEDIR_PEERICON_DIR)) create_dir(os.path.join(scfg.get_state_dir(), u"sqlite")) create_dir(os.path.join(scfg.get_state_dir(), STATEDIR_DLPSTATE_DIR)) # Reset the nickname to something not related to the host name, it was # really silly to have this default on the first place. # TODO: Maybe move this to the upgrader? if socket.gethostname() in scfg.get_nickname(): scfg.set_nickname("Tribler user") if GOTM2CRYPTO: permidmod.init() # Set params that depend on state_dir # # 1. keypair # pairfilename = scfg.get_permid_keypair_filename() if os.access(pairfilename, os.F_OK): # May throw exceptions self.keypair = permidmod.read_keypair(pairfilename) else: self.keypair = permidmod.generate_keypair() # Save keypair pubfilename = os.path.join(scfg.get_state_dir(), 'ecpub.pem') permidmod.save_keypair(self.keypair, pairfilename) permidmod.save_pub_key(self.keypair, pubfilename) if not scfg.get_megacache(): scfg.set_torrent_checking(0) self.sessconfig = scfg.sessconfig self.sessconfig.lock = self.sesslock self.selected_ports = scfg.selected_ports # Claim all random ports self.get_listen_port() self.get_dispersy_port() self.get_mainline_dht_listen_port() self.get_videoplayer_port() self.get_anon_listen_port() self.get_tunnel_community_socks5_listen_ports() # Create handler for calling back the user via separate threads self.lm = TriblerLaunchMany() self.notifier = Notifier(use_pool=True) # Checkpoint startup config self.save_pstate_sessconfig() self.sqlite_db = None self.autoload_discovery = autoload_discovery def prestart(self): """ Pre-starts the session. We check the current version and upgrade if needed - before we start everything else. """ self.sqlite_db = SQLiteCacheDB(self) self.sqlite_db.initialize() self.sqlite_db.initial_begin() self.upgrader = TriblerUpgrader(self, self.sqlite_db) self.upgrader.run() return self.upgrader # # Class methods # @staticmethod def get_instance(*args, **kw): """ Returns the Session singleton if it exists or otherwise creates it first, in which case you need to pass the constructor params. @return Session.""" if Session.__single is None: Session(*args, **kw) return Session.__single @staticmethod def has_instance(): return Session.__single is not None @staticmethod def del_instance(): Session.__single = None # # Public methods # def start_download(self, tdef, dcfg=None, initialdlstatus=None, hidden=False): """ Creates a Download object and adds it to the session. The passed ContentDef and DownloadStartupConfig are copied into the new Download object. The Download is then started and checkpointed. If a checkpointed version of the Download is found, that is restarted overriding the saved DownloadStartupConfig if "dcfg" is not None. @param tdef A finalized TorrentDef @param dcfg DownloadStartupConfig or None, in which case a new DownloadStartupConfig() is created with its default settings and the result becomes the runtime config of this Download. @param initialdlstatus The initial download status of this Download or None. This enables the caller to create a Download in e.g. DLSTATUS_STOPPED state instead. @param hidden Whether this torrent should be added to the mypreference table @return Download """ # locking by lm if self.get_libtorrent(): return self.lm.add(tdef, dcfg, initialdlstatus=initialdlstatus, hidden=hidden) raise OperationNotEnabledByConfigurationException() def resume_download_from_file(self, filename): """ Recreates Download from resume file @return a Download object. Note: this cannot be made into a method of Download, as the Download needs to be bound to a session, it cannot exist independently. """ raise NotYetImplementedException() def get_downloads(self): """ Returns a copy of the list of Downloads. @return A list of Download objects. """ # locking by lm return self.lm.get_downloads() def get_download(self, infohash): """ Returns the Download object for this hash. @return A Donwload Object. """ # locking by lm return self.lm.get_download(infohash) def has_download(self, infohash): """ Checks if the torrent download already exists. :param infohash: The torrent infohash. :return: True or False indicating if the torrent download already exists. """ return self.lm.download_exists(infohash) def remove_download(self, d, removecontent=False, removestate=True, hidden=False): """ Stops the download and removes it from the session. @param d The Download to remove @param removecontent Whether to delete the already downloaded content from disk. @param removestate Whether to delete the metadata files of the downloaded content from disk. @param hidden Whether this torrent is added to the mypreference table and this entry should be removed """ # locking by lm self.lm.remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden) def remove_download_by_id(self, infohash, removecontent=False, removestate=True): """ @param infohash The Download to remove @param removecontent Whether to delete the already downloaded content from disk. !We can only remove content when the download object is found, otherwise only the state is removed. """ downloadList = self.get_downloads() for download in downloadList: if download.get_def().get_infohash() == infohash: self.remove_download(download, removecontent, removestate) return self.lm.remove_id(infohash) def set_download_states_callback(self, usercallback, getpeerlist=None): """ See Download.set_state_callback. Calls usercallback with a list of DownloadStates, one for each Download in the Session as first argument. The usercallback must return a tuple (when,getpeerlist) that indicates when to reinvoke the callback again (as a number of seconds from now, or < 0.0 if not at all) and whether to also include the details of the connected peers in the DownloadStates on that next call. The callback will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. @param usercallback A function adhering to the above spec. """ self.lm.set_download_states_callback(usercallback, getpeerlist or []) # # Config parameters that only exist at runtime # def get_permid(self): """ Returns the PermID of the Session, as determined by the SessionConfig.set_permid() parameter. A PermID is a public key @return The PermID encoded in a string in DER format. """ return str(self.keypair.pub().get_der()) def get_current_startup_config_copy(self): """ Returns a SessionStartupConfig that is a copy of the current runtime SessionConfig. @return SessionStartupConfig """ # Called by any thread with self.sesslock: sessconfig = copy.copy(self.sessconfig) sessconfig.set_callback(None) return SessionStartupConfig(sessconfig=sessconfig) # # Notification of events in the Session # def add_observer(self, func, subject, changeTypes=[NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID=None, cache=0): """ Add an observer function function to the Session. The observer function will be called when one of the specified events (changeTypes) occurs on the specified subject. The function will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. @param func The observer function. It should accept as its first argument the subject, as second argument the changeType, as third argument an objectID (e.g. the primary key in the observed database) and an optional list of arguments. @param subject The subject to observe, one of NTFY_* subjects (see simpledefs). @param changeTypes The list of events to be notified of one of NTFY_* events. @param objectID The specific object in the subject to monitor (e.g. a specific primary key in a database to monitor for updates.) @param cache The time to bundle/cache events matching this function TODO: Jelle will add per-subject/event description here ;o) """ # Called by any thread self.notifier.add_observer(func, subject, changeTypes, objectID, cache=cache) # already threadsafe def remove_observer(self, func): """ Remove observer function. No more callbacks will be made. @param func The observer function to remove. """ # Called by any thread self.notifier.remove_observer(func) # already threadsafe def open_dbhandler(self, subject): """ Opens a connection to the specified database. Only the thread calling this method may use this connection. The connection must be closed with close_dbhandler() when this thread exits. @param subject The database to open. Must be one of the subjects specified here. @return A reference to a DBHandler class for the specified subject or None when the Session was not started with megacaches enabled. <pre> NTFY_PEERS -> PeerDBHandler NTFY_TORRENTS -> TorrentDBHandler NTFY_MYPREFERENCES -> MyPreferenceDBHandler NTFY_VOTECAST -> VotecastDBHandler NTFY_CHANNELCAST -> ChannelCastDBHandler </pre> """ if not self.get_megacache(): raise OperationNotEnabledByConfigurationException() # Called by any thread if subject == NTFY_METADATA: return self.lm.metadata_db elif subject == NTFY_PEERS: return self.lm.peer_db elif subject == NTFY_TORRENTS: return self.lm.torrent_db elif subject == NTFY_MYPREFERENCES: return self.lm.mypref_db elif subject == NTFY_VOTECAST: return self.lm.votecast_db elif subject == NTFY_CHANNELCAST: return self.lm.channelcast_db else: raise ValueError(u"Cannot open DB subject: %s" % subject) def close_dbhandler(self, dbhandler): """ Closes the given database connection """ dbhandler.close() def get_statistics(self): from Tribler.Core.statistics import TriblerStatistics return TriblerStatistics(self).dump_statistics() # # Persistence and shutdown # def load_checkpoint(self, initialdlstatus=None, initialdlstatus_dict={}): """ Restart Downloads from checkpoint, if any. This method allows the API user to manage restoring downloads. E.g. a video player that wants to start the torrent the user clicked on first, and only then restart any sleeping torrents (e.g. seeding). The optional initialdlstatus parameter can be set to DLSTATUS_STOPPED to restore all the Downloads in DLSTATUS_STOPPED state. The options initialdlstatus_dict parameter can be used to specify a state overriding the initaldlstatus parameter per download id. """ self.lm.load_checkpoint(initialdlstatus, initialdlstatus_dict) def checkpoint(self): """ Saves the internal session state to the Session's state dir. """ # Called by any thread self.checkpoint_shutdown(stop=False, checkpoint=True, gracetime=None, hacksessconfcheckpoint=False) def start(self): """ Create the LaunchManyCore instance and start it""" # Create engine with network thread self.lm.register(self, self.sesslock, autoload_discovery=self.autoload_discovery) self.sessconfig.set_callback(self.lm.sessconfig_changed_callback) def shutdown(self, checkpoint=True, gracetime=2.0, hacksessconfcheckpoint=True): """ Checkpoints the session and closes it, stopping the download engine. @param checkpoint Whether to checkpoint the Session state on shutdown. @param gracetime Time to allow for graceful shutdown + signoff (seconds). """ # Called by any thread self.lm.early_shutdown() self.checkpoint_shutdown(stop=True, checkpoint=checkpoint, gracetime=gracetime, hacksessconfcheckpoint=hacksessconfcheckpoint) self.sqlite_db.close() self.sqlite_db = None def has_shutdown(self): """ Whether the Session has completely shutdown, i.e., its internal threads are finished and it is safe to quit the process the Session is running in. @return A Boolean. """ return self.lm.sessdoneflag.isSet() def get_downloads_pstate_dir(self): """ Returns the directory in which to checkpoint the Downloads in this Session. """ # Called by network thread return os.path.join(self.get_state_dir(), STATEDIR_DLPSTATE_DIR) def download_torrentfile(self, infohash=None, usercallback=None, prio=0): """ Try to download the torrentfile without a known source. A possible source could be the DHT. If the torrent is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(None, infohash, user_callback=usercallback, priority=prio) def download_torrentfile_from_peer(self, candidate, infohash=None, usercallback=None, prio=0): """ Ask the designated peer to send us the torrentfile for the torrent identified by the passed infohash. If the torrent is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param permid The PermID of the peer to query. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(candidate, infohash, user_callback=usercallback, priority=prio) def download_torrentmessage_from_peer(self, candidate, infohash, usercallback, prio=0): """ Ask the designated peer to send us the torrentmessage for the torrent identified by the passed infohash. If the torrentmessage is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param permid The PermID of the peer to query. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrentmessage(candidate, infohash, usercallback, prio) def get_dispersy_instance(self): if not self.get_dispersy(): raise OperationNotEnabledByConfigurationException() return self.lm.dispersy def get_libtorrent_process(self): if not self.get_libtorrent(): raise OperationNotEnabledByConfigurationException() return self.lm.ltmgr # # Internal persistence methods # def checkpoint_shutdown(self, stop, checkpoint, gracetime, hacksessconfcheckpoint): """ Checkpoints the Session and optionally shuts down the Session. @param stop Whether to shutdown the Session as well. @param checkpoint Whether to checkpoint at all, or just to stop. @param gracetime Time to allow for graceful shutdown + signoff (seconds). """ # Called by any thread with self.sesslock: # Arno: Make checkpoint optional on shutdown. At the moment setting # the config at runtime is not possible (see SessionRuntimeConfig) # so this has little use, and interferes with our way of # changing the startup config, which is to write a new # config to disk that will be read at start up. if hacksessconfcheckpoint: try: self.save_pstate_sessconfig() except Exception as e: self._logger.error("save_pstate_sessconfig() failed with error: %s", e) # Checkpoint all Downloads and stop NetworkThread if stop: self._logger.debug("Session: checkpoint_shutdown") self.lm.checkpoint(stop=stop, checkpoint=checkpoint, gracetime=gracetime) def save_pstate_sessconfig(self): """ Save the runtime SessionConfig to disk """ # Called by any thread sscfg = self.get_current_startup_config_copy() cfgfilename = Session.get_default_config_filename(sscfg.get_state_dir()) sscfg.save(cfgfilename) def update_trackers(self, infohash, trackers): """ Updates the trackers of a torrent. :param infohash: infohash of the torrent that needs to be updated :param trackers: A list of tracker urls. """ return self.lm.update_trackers(infohash, trackers) # New APIs def has_collected_torrent(self, infohash): """ Checks if the given torrent infohash exists in the torrent_store database. :param infohash: The given infohash binary. :return: True or False indicating if we have the torrent. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") return hexlify(infohash) in self.lm.torrent_store def get_collected_torrent(self, infohash): """ Gets the given torrent from the torrent_store database. :param infohash: The given infohash binary. :return: The torrent data if exists, None otherwise. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") return self.lm.torrent_store.get(hexlify(infohash)) def save_collected_torrent(self, infohash, data): """ Saves the given torrent into the torrent_store database. :param infohash: The given infohash binary. :param data: The torrent file data. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") self.lm.torrent_store.put(hexlify(infohash), data) def search_remote_torrents(self, keywords): """ Searches for remote torrents through SearchCommunity with the given keywords. :param keywords: The given keywords. :return: The number of requests made. """ if not self.get_enable_torrent_search(): raise OperationNotEnabledByConfigurationException("torrent_search is not enabled") return self.lm.search_manager.search_for_torrents(keywords) def search_remote_channels(self, keywords): """ Searches for remote channels through AllChannelCommunity with the given keywords. :param keywords: The given keywords. """ if not self.get_enable_channel_search(): raise OperationNotEnabledByConfigurationException("channel_search is not enabled") self.lm.search_manager.search_for_channels(keywords) def create_channel(self, name, description, mode=u'open'): """ Creates a new Channel. :param name: Name of the Channel. :param description: Description of the Channel. :param mode: Mode of the Channel ('open', 'semi-open', or 'closed'). """ if not self.get_enable_channel_search(): raise OperationNotEnabledByConfigurationException("channel_search is not enabled") self.lm.channel_manager.create_channel(name, description, mode) def check_torrent_health(self, infohash): """ Checks the given torrent's health on its trackers. :param infohash: The given torrent infohash. """ self.lm.torrent_checker.add_gui_request(infohash) def set_max_upload_speed(self, rate): """ Sets the maximum upload rate (kB/s). :param rate: The upload rate (kB/s). """ if not self.get_libtorrent(): raise OperationNotEnabledByConfigurationException("libtorrent is not enabled") self.lm.ltmgr.set_upload_rate_limit(rate) def set_max_download_speed(self, rate): """ Sets the maximum download rate (kB/s). :param rate: The download rate (kB/s). """ if not self.lm.ltmgr: raise OperationNotEnabledByConfigurationException("libtorrent is not enabled") self.lm.ltmgr.set_download_rate_limit(rate)
class TestSqliteCacheDB(AbstractServer): def setUp(self): super(TestSqliteCacheDB, self).setUp() self.config = SessionStartupConfig() self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking(False) self.config.set_multicast_local_peer_discovery(False) self.config.set_megacache(False) self.config.set_dispersy(False) self.config.set_mainline_dht(False) self.config.set_torrent_collecting(False) self.config.set_libtorrent(False) self.config.set_dht_torrent_collecting(False) self.config.set_videoplayer(False) self.session = Session(self.config, ignore_singleton=True) self.sqlite_test = SQLiteCacheDB(self.session) self.db_path = u":memory:" self.sqlite_test.initialize(self.db_path) def tearDown(self): super(TestSqliteCacheDB, self).tearDown() self.sqlite_test.close() self.sqlite_test = None self.session.del_instance() self.session = None @blocking_call_on_reactor_thread def test_create_db(self): sql = u"CREATE TABLE person(lastname, firstname);" self.sqlite_test.execute(sql) @blocking_call_on_reactor_thread def test_insert(self): self.test_create_db() self.sqlite_test.insert('person', lastname='a', firstname='b') assert self.sqlite_test.size('person') == 1 @blocking_call_on_reactor_thread def test_fetchone(self): self.test_insert() one = self.sqlite_test.fetchone(u"SELECT * FROM person") assert one == ('a', 'b') one = self.sqlite_test.fetchone(u"SELECT lastname FROM person WHERE firstname == 'b'") assert one == 'a' one = self.sqlite_test.fetchone(u"SELECT lastname FROM person WHERE firstname == 'c'") assert one is None @blocking_call_on_reactor_thread def test_insertmany(self): self.test_create_db() values = [] for i in range(100): value = (str(i), str(i ** 2)) values.append(value) self.sqlite_test.insertMany('person', values) assert self.sqlite_test.size('person') == 100 @blocking_call_on_reactor_thread def test_fetchall(self): self.test_insertmany() all = self.sqlite_test.fetchall('select * from person') assert len(all) == 100 all = self.sqlite_test.fetchall("select * from person where lastname=='101'") assert all == [] @blocking_call_on_reactor_thread def test_insertorder(self): self.test_insertmany() self.sqlite_test.insert('person', lastname='1', firstname='abc') one = self.sqlite_test.fetchone("select firstname from person where lastname == '1'") assert one == '1' or one == 'abc' all = self.sqlite_test.fetchall("select firstname from person where lastname == '1'") assert len(all) == 2 @blocking_call_on_reactor_thread def test_update(self): self.test_insertmany() self.sqlite_test.update('person', "lastname == '2'", firstname='56') one = self.sqlite_test.fetchone("select firstname from person where lastname == '2'") assert one == '56', one self.sqlite_test.update('person', "lastname == '3'", firstname=65) one = self.sqlite_test.fetchone("select firstname from person where lastname == '3'") assert one == 65, one self.sqlite_test.update('person', "lastname == '4'", firstname=654, lastname=44) one = self.sqlite_test.fetchone("select firstname from person where lastname == 44") assert one == 654, one
def test_no_version_info_in_database(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), os.path.join(self.SQLITE_SCRIPTS_DIR, "script1.sql")) sqlite_test_2.initialize()
def test_open_db_connection_no_permission(self): os.chmod(os.path.join(self.session_base_dir), 0) sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db")) sqlite_test_2.initialize()
def test_open_db_new_file(self): db_path = os.path.join(self.session_base_dir, "test_db.db") sqlite_test_2 = SQLiteCacheDB(db_path) sqlite_test_2.initialize() self.assertTrue(os.path.isfile(db_path))
class TestSqliteCacheDB(AbstractServer): def setUp(self): super(TestSqliteCacheDB, self).setUp() self.config = SessionStartupConfig() self.config.set_state_dir(self.getStateDir()) self.config.set_torrent_checking(False) self.config.set_multicast_local_peer_discovery(False) self.config.set_megacache(False) self.config.set_dispersy(False) self.config.set_mainline_dht(False) self.config.set_torrent_collecting(False) self.config.set_libtorrent(False) self.config.set_dht_torrent_collecting(False) self.config.set_videoplayer(False) self.session = Session(self.config, ignore_singleton=True) self.sqlite_test = SQLiteCacheDB(self.session) self.db_path = u":memory:" self.sqlite_test.initialize(self.db_path) def tearDown(self): super(TestSqliteCacheDB, self).tearDown() self.sqlite_test.close() self.sqlite_test = None self.session.del_instance() self.session = None @blocking_call_on_reactor_thread def test_create_db(self): sql = u"CREATE TABLE person(lastname, firstname);" self.sqlite_test.execute(sql) @blocking_call_on_reactor_thread def test_insert(self): self.test_create_db() self.sqlite_test.insert('person', lastname='a', firstname='b') assert self.sqlite_test.size('person') == 1 @blocking_call_on_reactor_thread def test_fetchone(self): self.test_insert() one = self.sqlite_test.fetchone(u"SELECT * FROM person") assert one == ('a', 'b') one = self.sqlite_test.fetchone( u"SELECT lastname FROM person WHERE firstname == 'b'") assert one == 'a' one = self.sqlite_test.fetchone( u"SELECT lastname FROM person WHERE firstname == 'c'") assert one is None @blocking_call_on_reactor_thread def test_insertmany(self): self.test_create_db() values = [] for i in range(100): value = (str(i), str(i**2)) values.append(value) self.sqlite_test.insertMany('person', values) assert self.sqlite_test.size('person') == 100 @blocking_call_on_reactor_thread def test_fetchall(self): self.test_insertmany() all = self.sqlite_test.fetchall('select * from person') assert len(all) == 100 all = self.sqlite_test.fetchall( "select * from person where lastname=='101'") assert all == [] @blocking_call_on_reactor_thread def test_insertorder(self): self.test_insertmany() self.sqlite_test.insert('person', lastname='1', firstname='abc') one = self.sqlite_test.fetchone( "select firstname from person where lastname == '1'") assert one == '1' or one == 'abc' all = self.sqlite_test.fetchall( "select firstname from person where lastname == '1'") assert len(all) == 2 @blocking_call_on_reactor_thread def test_update(self): self.test_insertmany() self.sqlite_test.update('person', "lastname == '2'", firstname='56') one = self.sqlite_test.fetchone( "select firstname from person where lastname == '2'") assert one == '56', one self.sqlite_test.update('person', "lastname == '3'", firstname=65) one = self.sqlite_test.fetchone( "select firstname from person where lastname == '3'") assert one == 65, one self.sqlite_test.update('person', "lastname == '4'", firstname=654, lastname=44) one = self.sqlite_test.fetchone( "select firstname from person where lastname == 44") assert one == 654, one
def test_no_file_db_error(self): file_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) sqlite_test_2 = SQLiteCacheDB(file_dir) sqlite_test_2.initialize()
class Session(SessionConfigInterface): """ A Session is a running instance of the Tribler Core and the Core's central class. It implements the SessionConfigInterface which can be used to change session parameters at runtime (for selected parameters). cf. libtorrent session """ __single = None def __init__(self, scfg=None, ignore_singleton=False, autoload_discovery=True): """ A Session object is created which is configured following a copy of the SessionStartupConfig scfg. (copy constructor used internally) @param scfg SessionStartupConfig object or None, in which case we look for a saved session in the default location (state dir). If we can't find it, we create a new SessionStartupConfig() object to serve as startup config. Next, the config is saved in the directory indicated by its 'state_dir' attribute. In the current implementation only a single session instance can exist at a time in a process. The ignore_singleton flag is used for testing. """ addObserver(self.unhandled_error_observer) patch_crypto_be_discovery() if not ignore_singleton: if Session.__single: raise RuntimeError("Session is singleton") Session.__single = self self._logger = logging.getLogger(self.__class__.__name__) self.ignore_singleton = ignore_singleton self.sesslock = NoDispersyRLock() # Determine startup config to use if scfg is None: # If no override scfg = SessionStartupConfig.load() else: # overrides any saved config # Work from copy scfg = SessionStartupConfig(copy.copy(scfg.sessconfig)) def create_dir(fullpath): if not os.path.isdir(fullpath): os.makedirs(fullpath) def set_and_create_dir(dirname, setter, default_dir): if dirname is None: setter(default_dir) create_dir(dirname or default_dir) state_dir = scfg.get_state_dir() set_and_create_dir(state_dir, scfg.set_state_dir, state_dir) set_and_create_dir( scfg.get_torrent_store_dir(), scfg.set_torrent_store_dir, os.path.join(scfg.get_state_dir(), STATEDIR_TORRENT_STORE_DIR)) # metadata store set_and_create_dir( scfg.get_metadata_store_dir(), scfg.set_metadata_store_dir, os.path.join(scfg.get_state_dir(), STATEDIR_METADATA_STORE_DIR)) set_and_create_dir( scfg.get_peer_icon_path(), scfg.set_peer_icon_path, os.path.join(scfg.get_state_dir(), STATEDIR_PEERICON_DIR)) create_dir(os.path.join(scfg.get_state_dir(), u"sqlite")) create_dir(os.path.join(scfg.get_state_dir(), STATEDIR_DLPSTATE_DIR)) if GOTM2CRYPTO: permidmod.init() # Set params that depend on state_dir # # 1. keypair # pairfilename = scfg.get_permid_keypair_filename() if os.path.exists(pairfilename): self.keypair = permidmod.read_keypair(pairfilename) else: self.keypair = permidmod.generate_keypair() # Save keypair pubfilename = os.path.join(scfg.get_state_dir(), 'ecpub.pem') permidmod.save_keypair(self.keypair, pairfilename) permidmod.save_pub_key(self.keypair, pubfilename) multichain_pairfilename = scfg.get_multichain_permid_keypair_filename( ) if os.path.exists(multichain_pairfilename): self.multichain_keypair = permidmod.read_keypair_multichain( multichain_pairfilename) else: self.multichain_keypair = permidmod.generate_keypair_multichain( ) # Save keypair multichain_pubfilename = os.path.join(scfg.get_state_dir(), 'ecpub_multichain.pem') permidmod.save_keypair_multichain(self.multichain_keypair, multichain_pairfilename) permidmod.save_pub_key_multichain(self.multichain_keypair, multichain_pubfilename) if not scfg.get_megacache(): scfg.set_torrent_checking(0) self.sessconfig = scfg.sessconfig self.sessconfig.lock = self.sesslock self.selected_ports = scfg.selected_ports # Claim all random ports self.get_listen_port() self.get_dispersy_port() self.get_mainline_dht_listen_port() self.get_videoserver_port() self.get_anon_listen_port() self.get_tunnel_community_socks5_listen_ports() # Create handler for calling back the user via separate threads self.lm = TriblerLaunchMany() self.notifier = Notifier() # Checkpoint startup config self.save_session_config() self.sqlite_db = None self.upgrader_enabled = True self.dispersy_member = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.autoload_discovery = autoload_discovery self.tribler_config = TriblerConfig(self) self.setup_tribler_gui_config() # # Class methods # @staticmethod def get_instance(*args, **kw): """ Returns the Session singleton if it exists or otherwise creates it first, in which case you need to pass the constructor params. @return Session.""" if Session.__single is None: Session(*args, **kw) return Session.__single @staticmethod def has_instance(): return Session.__single is not None @staticmethod def del_instance(): Session.__single = None def unhandled_error_observer(self, event): """ This method is called when an unhandled error in Tribler is observed. Broadcasts the tribler_exception event. """ if event['isError']: text = "" if 'log_legacy' in event and 'log_text' in event: text = event['log_text'] elif 'log_failure' in event: text = str(event['log_failure']) # There are some errors that we are ignoring. # No route to host: this issue is non-critical since Tribler can still function when a request fails. if 'socket.error: [Errno 113]' in text: self._logger.error( "Observed no route to host error (but ignoring)." "This might indicate a problem with your firewall.") return # Socket block: this sometimes occurres on Windows and is non-critical. if 'socket.error: [Errno %s]' % SOCKET_BLOCK_ERRORCODE in text: self._logger.error( "Unable to send data due to socket.error %s", SOCKET_BLOCK_ERRORCODE) return if 'socket.error: [Errno 51]' in text: self._logger.error( "Could not send data: network is unreachable.") return if 'exceptions.ValueError: Invalid DNS-ID' in text: self._logger.error("Invalid DNS-ID") return # We already have a check for invalid infohash when adding a torrent, but if somehow we get this # error then we simply log and ignore it. if 'exceptions.RuntimeError: invalid info-hash' in text: self._logger.error("Invalid info-hash found") return if self.lm.api_manager and len(text) > 0: self.lm.api_manager.root_endpoint.events_endpoint.on_tribler_exception( text) self.lm.api_manager.root_endpoint.state_endpoint.on_tribler_exception( text) # # Public methods # def setup_tribler_gui_config(self): """ Initialize the TriblerGUI configuration file and make sure that we have all required values. """ configfilepath = os.path.join(self.get_state_dir(), STATEDIR_GUICONFIG) gui_config = CallbackConfigParser() DefaultDownloadStartupConfig.getInstance().set_dest_dir( get_default_dest_dir()) # Load the config file. if os.path.exists(configfilepath): gui_config.read_file(configfilepath, 'utf-8-sig') if not gui_config.has_section('Tribler'): gui_config.add_section('Tribler') for k, v in tribler_defaults['Tribler'].iteritems(): if not gui_config.has_option('Tribler', k): gui_config.set('Tribler', k, v) if not gui_config.has_section('downloadconfig'): gui_config.add_section('downloadconfig') for k, v in DefaultDownloadStartupConfig.getInstance( ).dlconfig._sections['downloadconfig'].iteritems(): if not gui_config.has_option('downloadconfig', k): gui_config.set('downloadconfig', k, v) # Make sure we use the same ConfigParser instance for both Utility and DefaultDownloadStartupConfig. DefaultDownloadStartupConfig.getInstance().dlconfig = gui_config gui_config.write_file(configfilepath) # Update all dldefaults to use the settings in gui_config for k, v in gui_config._sections['downloadconfig'].iteritems(): dldefaults['downloadconfig'][k] = v def start_download_from_uri(self, uri, dconfig=None): """ Start a download from an argument. This argument can be of the following type: -http: Start a download from a torrent file at the given url. -magnet: Start a download from a torrent file by using a magnet link. -file: Start a download from a torrent file at given location. :param uri: The argument that specifies the location of the torrent to be downloaded. :param dconfig: An optional configuration for the download. :return: A deferred that fires when a download has been added to the Tribler core. """ if self.get_libtorrent(): return self.lm.ltmgr.start_download_from_uri(uri, dconfig=dconfig) raise OperationNotEnabledByConfigurationException() def start_download_from_tdef(self, tdef, dcfg=None, hidden=False): """ Creates a Download object and adds it to the session. The passed ContentDef and DownloadStartupConfig are copied into the new Download object. The Download is then started and checkpointed. If a checkpointed version of the Download is found, that is restarted overriding the saved DownloadStartupConfig if "dcfg" is not None. @param tdef A finalized TorrentDef @param dcfg DownloadStartupConfig or None, in which case a new DownloadStartupConfig() is created with its default settings and the result becomes the runtime config of this Download. @param hidden Whether this torrent should be added to the mypreference table @return Download """ # locking by lm if self.get_libtorrent(): return self.lm.add(tdef, dcfg, hidden=hidden) raise OperationNotEnabledByConfigurationException() def resume_download_from_file(self, filename): """ Recreates Download from resume file @return a Download object. Note: this cannot be made into a method of Download, as the Download needs to be bound to a session, it cannot exist independently. """ raise NotYetImplementedException() def get_downloads(self): """ Returns a copy of the list of Downloads. @return A list of Download objects. """ # locking by lm return self.lm.get_downloads() def get_download(self, infohash): """ Returns the Download object for this hash. @return A Donwload Object. """ # locking by lm return self.lm.get_download(infohash) def has_download(self, infohash): """ Checks if the torrent download already exists. :param infohash: The torrent infohash. :return: True or False indicating if the torrent download already exists. """ return self.lm.download_exists(infohash) def remove_download(self, d, removecontent=False, removestate=True, hidden=False): """ Stops the download and removes it from the session. @param d The Download to remove @param removecontent Whether to delete the already downloaded content from disk. @param removestate Whether to delete the metadata files of the downloaded content from disk. @param hidden Whether this torrent is added to the mypreference table and this entry should be removed """ # locking by lm return self.lm.remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden) def remove_download_by_id(self, infohash, removecontent=False, removestate=True): """ @param infohash The Download to remove @param removecontent Whether to delete the already downloaded content from disk. !We can only remove content when the download object is found, otherwise only the state is removed. """ downloadList = self.get_downloads() for download in downloadList: if download.get_def().get_infohash() == infohash: return self.remove_download(download, removecontent, removestate) self.lm.remove_id(infohash) def set_download_states_callback(self, usercallback, interval=1.0): """ See Download.set_state_callback. Calls usercallback with a list of DownloadStates, one for each Download in the Session as first argument. The usercallback must return a tuple (when,getpeerlist) that indicates when to reinvoke the callback again (as a number of seconds from now, or < 0.0 if not at all) and whether to also include the details of the connected peers in the DownloadStates on that next call. The callback will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. @param usercallback A function adhering to the above spec. """ self.lm.set_download_states_callback(usercallback, interval) # # Config parameters that only exist at runtime # def get_permid(self): """ Returns the PermID of the Session, as determined by the SessionConfig.set_permid() parameter. A PermID is a public key @return The PermID encoded in a string in DER format. """ return str(self.keypair.pub().get_der()) def get_current_startup_config_copy(self): """ Returns a SessionStartupConfig that is a copy of the current runtime SessionConfig. @return SessionStartupConfig """ # Called by any thread with self.sesslock: sessconfig = copy.copy(self.sessconfig) sessconfig.set_callback(None) return SessionStartupConfig(sessconfig=sessconfig) # # Notification of events in the Session # def add_observer(self, func, subject, changeTypes=[NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID=None, cache=0): """ Add an observer function function to the Session. The observer function will be called when one of the specified events (changeTypes) occurs on the specified subject. The function will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. @param func The observer function. It should accept as its first argument the subject, as second argument the changeType, as third argument an objectID (e.g. the primary key in the observed database) and an optional list of arguments. @param subject The subject to observe, one of NTFY_* subjects (see simpledefs). @param changeTypes The list of events to be notified of one of NTFY_* events. @param objectID The specific object in the subject to monitor (e.g. a specific primary key in a database to monitor for updates.) @param cache The time to bundle/cache events matching this function TODO: Jelle will add per-subject/event description here ;o) """ # Called by any thread self.notifier.add_observer(func, subject, changeTypes, objectID, cache=cache) # already threadsafe def remove_observer(self, func): """ Remove observer function. No more callbacks will be made. @param func The observer function to remove. """ # Called by any thread self.notifier.remove_observer(func) # already threadsafe def open_dbhandler(self, subject): """ Opens a connection to the specified database. Only the thread calling this method may use this connection. The connection must be closed with close_dbhandler() when this thread exits. @param subject The database to open. Must be one of the subjects specified here. @return A reference to a DBHandler class for the specified subject or None when the Session was not started with megacaches enabled. <pre> NTFY_PEERS -> PeerDBHandler NTFY_TORRENTS -> TorrentDBHandler NTFY_MYPREFERENCES -> MyPreferenceDBHandler NTFY_VOTECAST -> VotecastDBHandler NTFY_CHANNELCAST -> ChannelCastDBHandler </pre> """ if not self.get_megacache(): raise OperationNotEnabledByConfigurationException() # Called by any thread if subject == NTFY_PEERS: return self.lm.peer_db elif subject == NTFY_TORRENTS: return self.lm.torrent_db elif subject == NTFY_MYPREFERENCES: return self.lm.mypref_db elif subject == NTFY_VOTECAST: return self.lm.votecast_db elif subject == NTFY_CHANNELCAST: return self.lm.channelcast_db else: raise ValueError(u"Cannot open DB subject: %s" % subject) def close_dbhandler(self, dbhandler): """ Closes the given database connection """ dbhandler.close() def get_tribler_statistics(self): """ Return a dictionary with general Tribler statistics. """ return TriblerStatistics(self).get_tribler_statistics() def get_dispersy_statistics(self): """ Return a dictionary with general Dispersy statistics. """ return TriblerStatistics(self).get_dispersy_statistics() def get_community_statistics(self): """ Return a dictionary with general communities statistics. """ return TriblerStatistics(self).get_community_statistics() # # Persistence and shutdown # def load_checkpoint(self): """ Restart Downloads from a saved checkpoint, if any. Note that we fetch information from the user download choices since it might be that a user has stopped a download. In that case, the download should not be resumed immediately when being loaded by libtorrent. """ self.lm.load_checkpoint() @blocking_call_on_reactor_thread def start_database(self): """ Start the SQLite database. """ db_path = os.path.join(self.get_state_dir(), DB_FILE_RELATIVE_PATH) db_script_path = os.path.join(get_lib_path(), DB_SCRIPT_NAME) self.sqlite_db = SQLiteCacheDB(db_path, db_script_path) self.readable_status = STATE_OPEN_DB self.sqlite_db.initialize() self.sqlite_db.initial_begin() @blocking_call_on_reactor_thread def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. """ # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.get_http_api_enabled(): self.lm.api_manager = RESTManager(self) self.readable_status = STATE_START_API self.lm.api_manager.start() self.start_database() # We clean the mugshot since it isn't used anymore and often contains data unsuitable for sending over the API. self.set_mugshot(None) if self.upgrader_enabled: upgrader = TriblerUpgrader(self, self.sqlite_db) self.readable_status = STATE_UPGRADING_READABLE upgrader.run() startup_deferred = self.lm.register(self, self.sesslock) def load_checkpoint(_): if self.get_libtorrent(): self.readable_status = STATE_LOAD_CHECKPOINTS self.load_checkpoint() self.readable_status = STATE_READABLE_STARTED self.sessconfig.set_callback(self.lm.sessconfig_changed_callback) return startup_deferred.addCallback(load_checkpoint) @blocking_call_on_reactor_thread def shutdown(self): """ Checkpoints the session and closes it, stopping the download engine. This method has to be called from the reactor thread. """ assert isInIOThread() @inlineCallbacks def on_early_shutdown_complete(_): """ Callback that gets called when the early shutdown has been completed. Continues the shutdown procedure that is dependant on the early shutdown. :param _: ignored parameter of the Deferred """ self.save_session_config() yield self.checkpoint_downloads() self.lm.shutdown_downloads() self.lm.network_shutdown() if self.sqlite_db: self.sqlite_db.close() self.sqlite_db = None return self.lm.early_shutdown().addCallback(on_early_shutdown_complete) def has_shutdown(self): """ Whether the Session has completely shutdown, i.e., its internal threads are finished and it is safe to quit the process the Session is running in. @return A Boolean. """ return self.lm.sessdoneflag.isSet() def get_downloads_pstate_dir(self): """ Returns the directory in which to checkpoint the Downloads in this Session. """ # Called by network thread return os.path.join(self.get_state_dir(), STATEDIR_DLPSTATE_DIR) def download_torrentfile(self, infohash=None, usercallback=None, prio=0): """ Try to download the torrentfile without a known source. A possible source could be the DHT. If the torrent is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(None, infohash, user_callback=usercallback, priority=prio) def download_torrentfile_from_peer(self, candidate, infohash=None, usercallback=None, prio=0): """ Ask the designated peer to send us the torrentfile for the torrent identified by the passed infohash. If the torrent is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param permid The PermID of the peer to query. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(candidate, infohash, user_callback=usercallback, priority=prio) def download_torrentmessage_from_peer(self, candidate, infohash, usercallback, prio=0): """ Ask the designated peer to send us the torrentmessage for the torrent identified by the passed infohash. If the torrentmessage is succesfully received, the usercallback method is called with the infohash as first and the contents of the torrentfile (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. @param permid The PermID of the peer to query. @param infohash The infohash of the torrent. @param usercallback A function adhering to the above spec. """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrentmessage( candidate, infohash, usercallback, prio) def get_dispersy_instance(self): if not self.get_dispersy(): raise OperationNotEnabledByConfigurationException() return self.lm.dispersy def get_libtorrent_process(self): if not self.get_libtorrent(): raise OperationNotEnabledByConfigurationException() return self.lm.ltmgr # # Internal persistence methods # def checkpoint_downloads(self): """ Checkpoints the downloads in Tribler. """ return self.lm.checkpoint_downloads() def save_session_config(self): """ Save the runtime SessionConfig to disk """ # Called by any thread sscfg = self.get_current_startup_config_copy() cfgfilename = Session.get_default_config_filename( sscfg.get_state_dir()) sscfg.save(cfgfilename) def update_trackers(self, infohash, trackers): """ Updates the trackers of a torrent. :param infohash: infohash of the torrent that needs to be updated :param trackers: A list of tracker urls. """ return self.lm.update_trackers(infohash, trackers) # New APIs def has_collected_torrent(self, infohash): """ Checks if the given torrent infohash exists in the torrent_store database. :param infohash: The given infohash binary. :return: True or False indicating if we have the torrent. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException( "torrent_store is not enabled") return hexlify(infohash) in self.lm.torrent_store def get_collected_torrent(self, infohash): """ Gets the given torrent from the torrent_store database. :param infohash: The given infohash binary. :return: The torrent data if exists, None otherwise. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException( "torrent_store is not enabled") return self.lm.torrent_store.get(hexlify(infohash)) def save_collected_torrent(self, infohash, data): """ Saves the given torrent into the torrent_store database. :param infohash: The given infohash binary. :param data: The torrent file data. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException( "torrent_store is not enabled") self.lm.torrent_store.put(hexlify(infohash), data) def delete_collected_torrent(self, infohash): """ Deletes the given torrent from the torrent_store database. :param infohash: The given infohash binary. """ if not self.get_torrent_store(): raise OperationNotEnabledByConfigurationException( "torrent_store is not enabled") del self.lm.torrent_store[hexlify(infohash)] def search_remote_torrents(self, keywords): """ Searches for remote torrents through SearchCommunity with the given keywords. :param keywords: The given keywords. :return: The number of requests made. """ if not self.get_enable_torrent_search(): raise OperationNotEnabledByConfigurationException( "torrent_search is not enabled") return self.lm.search_manager.search_for_torrents(keywords) def search_remote_channels(self, keywords): """ Searches for remote channels through AllChannelCommunity with the given keywords. :param keywords: The given keywords. """ if not self.get_enable_channel_search(): raise OperationNotEnabledByConfigurationException( "channel_search is not enabled") self.lm.search_manager.search_for_channels(keywords) def create_torrent_file(self, file_path_list, params={}): """ :param file_path_list: files to add in torrent file :param params: optional parameters for torrent file :return: Deferred """ return threads.deferToThread(torrent_utils.create_torrent_file, file_path_list, params) def create_channel(self, name, description, mode=u'closed'): """ Creates a new Channel. :param name: Name of the Channel. :param description: Description of the Channel. :param mode: Mode of the Channel ('open', 'semi-open', or 'closed'). :return: Channel ID :raises DuplicateChannelNameError if name already exists """ return self.lm.channel_manager.create_channel(name, description, mode) def add_torrent_def_to_channel(self, channel_id, torrent_def, extra_info={}, forward=True): """ Adds a TorrentDef to a Channel. :param channel_id: Id of the Channel to add the Torrent to. :param torrent_def: Definition of the Torrent to add. :param extra_info: Description of the Torrent to add. :param forward: When True the messages are forwarded (as defined by their message destination policy) to other nodes in the community. This parameter should (almost always) be True, its inclusion is mostly to allow certain debugging scenarios. """ # Make sure that this new torrent_def is also in collected torrents self.lm.rtorrent_handler.save_torrent(torrent_def) channelcast_db = self.open_dbhandler(NTFY_CHANNELCAST) if channelcast_db.hasTorrent(channel_id, torrent_def.infohash): raise DuplicateTorrentFileError( "This torrent file already exists in your channel.") dispersy_cid = str( channelcast_db.getDispersyCIDFromChannelId(channel_id)) community = self.get_dispersy_instance().get_community(dispersy_cid) community._disp_create_torrent( torrent_def.infohash, long(time.time()), torrent_def.get_name_as_unicode(), tuple(torrent_def.get_files_with_length()), torrent_def.get_trackers_as_single_tuple(), forward=forward) if 'description' in extra_info: desc = extra_info['description'].strip() if desc != '': data = channelcast_db.getTorrentFromChannelId( channel_id, torrent_def.infohash, ['ChannelTorrents.id']) community.modifyTorrent(data, {'description': desc}, forward=forward) def check_torrent_health(self, infohash, timeout=20, scrape_now=False): """ Checks the given torrent's health on its trackers. :param infohash: The given torrent infohash. """ if self.lm.torrent_checker: return self.lm.torrent_checker.add_gui_request( infohash, timeout=timeout, scrape_now=scrape_now) return fail(Failure(RuntimeError("Torrent checker not available"))) def set_max_upload_speed(self, rate): """ Sets the maximum upload rate (kB/s). :param rate: The upload rate (kB/s). """ if not self.get_libtorrent(): raise OperationNotEnabledByConfigurationException( "libtorrent is not enabled") self.lm.ltmgr.set_upload_rate_limit(rate) def set_max_download_speed(self, rate): """ Sets the maximum download rate (kB/s). :param rate: The download rate (kB/s). """ if not self.lm.ltmgr: raise OperationNotEnabledByConfigurationException( "libtorrent is not enabled") self.lm.ltmgr.set_download_rate_limit(rate) def get_thumbnail_data(self, thumb_hash): """ Gets the thumbnail data. :param thumb_hash: The thumbnail SHA1 hash. :return: The thumbnail data. """ if not self.lm.metadata_store: raise OperationNotEnabledByConfigurationException( "libtorrent is not enabled") return self.lm.rtorrent_handler.get_metadata(thumb_hash)
def test_open_db_script_file_invalid_location(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), u'myfakelocation') sqlite_test_2.initialize()
class Session(object): """ A Session is a running instance of the Tribler Core and the Core's central class. """ __single = None def __init__(self, config=None, autoload_discovery=True): """ A Session object is created which is configured with the Tribler configuration object. Only a single session instance can exist at a time in a process. :param config: a TriblerConfig object or None, in which case we look for a saved session in the default location (state dir). If we can't find it, we create a new TriblerConfig() object to serve as startup config. Next, the config is saved in the directory indicated by its 'state_dir' attribute. :param autoload_discovery: only false in the Tunnel community tests """ addObserver(self.unhandled_error_observer) patch_crypto_be_discovery() self._logger = logging.getLogger(self.__class__.__name__) self.session_lock = NoDispersyRLock() self.config = config or TriblerConfig() self.get_ports_in_config() self.create_state_directory_structure() if not self.config.get_megacache_enabled(): self.config.set_torrent_checking_enabled(False) self.selected_ports = self.config.selected_ports self.init_keypair() self.lm = TriblerLaunchMany() self.notifier = Notifier() self.sqlite_db = None self.upgrader_enabled = True self.dispersy_member = None self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler self.autoload_discovery = autoload_discovery def create_state_directory_structure(self): """Create directory structure of the state directory.""" def create_dir(path): if not os.path.isdir(path): os.makedirs(path) def create_in_state_dir(path): create_dir(os.path.join(self.config.get_state_dir(), path)) create_dir(self.config.get_state_dir()) create_dir(self.config.get_torrent_store_dir()) create_dir(self.config.get_metadata_store_dir()) create_in_state_dir(DB_DIR_NAME) create_in_state_dir(STATEDIR_DLPSTATE_DIR) create_in_state_dir(STATEDIR_WALLET_DIR) def get_ports_in_config(self): """Claim all required random ports.""" self.config.get_libtorrent_port() self.config.get_dispersy_port() self.config.get_mainline_dht_port() self.config.get_video_server_port() self.config.get_anon_listen_port() self.config.get_tunnel_community_socks5_listen_ports() def init_keypair(self): """ Set parameters that depend on state_dir. """ permid_module.init() # Set params that depend on state_dir # # 1. keypair # pair_filename = self.config.get_permid_keypair_filename() if os.path.exists(pair_filename): self.keypair = permid_module.read_keypair(pair_filename) else: self.keypair = permid_module.generate_keypair() # Save keypair public_key_filename = os.path.join(self.config.get_state_dir(), 'ecpub.pem') permid_module.save_keypair(self.keypair, pair_filename) permid_module.save_pub_key(self.keypair, public_key_filename) trustchain_pairfilename = self.config.get_trustchain_keypair_filename() if os.path.exists(trustchain_pairfilename): self.trustchain_keypair = permid_module.read_keypair_trustchain(trustchain_pairfilename) else: self.trustchain_keypair = permid_module.generate_keypair_trustchain() # Save keypair trustchain_pubfilename = os.path.join(self.config.get_state_dir(), 'ecpub_multichain.pem') permid_module.save_keypair_trustchain(self.trustchain_keypair, trustchain_pairfilename) permid_module.save_pub_key_trustchain(self.trustchain_keypair, trustchain_pubfilename) trustchain_testnet_pairfilename = self.config.get_trustchain_testnet_keypair_filename() if os.path.exists(trustchain_testnet_pairfilename): self.trustchain_testnet_keypair = permid_module.read_keypair_trustchain(trustchain_testnet_pairfilename) else: self.trustchain_testnet_keypair = permid_module.generate_keypair_trustchain() # Save keypair trustchain_testnet_pubfilename = os.path.join(self.config.get_state_dir(), 'ecpub_trustchain_testnet.pem') permid_module.save_keypair_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pairfilename) permid_module.save_pub_key_trustchain(self.trustchain_testnet_keypair, trustchain_testnet_pubfilename) def unhandled_error_observer(self, event): """ This method is called when an unhandled error in Tribler is observed. It broadcasts the tribler_exception event. """ if event['isError']: text = "" if 'log_legacy' in event and 'log_text' in event: text = event['log_text'] elif 'log_failure' in event: text = str(event['log_failure']) # There are some errors that we are ignoring. # No route to host: this issue is non-critical since Tribler can still function when a request fails. if 'socket.error' in text and '[Errno 113]' in text: self._logger.error("Observed no route to host error (but ignoring)." "This might indicate a problem with your firewall.") return # Socket block: this sometimes occurres on Windows and is non-critical. if 'socket.error' in text and '[Errno %s]' % SOCKET_BLOCK_ERRORCODE in text: self._logger.error("Unable to send data due to socket.error %s", SOCKET_BLOCK_ERRORCODE) return if 'socket.error' in text and '[Errno 51]' in text: self._logger.error("Could not send data: network is unreachable.") return if 'socket.error' in text and '[Errno 16]' in text: self._logger.error("Could not send data: socket is busy.") return if 'socket.error' in text and '[Errno 10054]' in text: self._logger.error("Connection forcibly closed by the remote host.") return if 'exceptions.ValueError: Invalid DNS-ID' in text: self._logger.error("Invalid DNS-ID") return if 'twisted.web._newclient.ResponseNeverReceived' in text: self._logger.error("Internal Twisted response error, consider updating your Twisted version.") return # We already have a check for invalid infohash when adding a torrent, but if somehow we get this # error then we simply log and ignore it. if 'exceptions.RuntimeError: invalid info-hash' in text: self._logger.error("Invalid info-hash found") return if self.lm.api_manager and len(text) > 0: self.lm.api_manager.root_endpoint.events_endpoint.on_tribler_exception(text) self.lm.api_manager.root_endpoint.state_endpoint.on_tribler_exception(text) def start_download_from_uri(self, uri, download_config=None): """ Start a download from an argument. This argument can be of the following type: -http: Start a download from a torrent file at the given url. -magnet: Start a download from a torrent file by using a magnet link. -file: Start a download from a torrent file at given location. :param uri: specifies the location of the torrent to be downloaded :param download_config: an optional configuration for the download :return: a deferred that fires when a download has been added to the Tribler core """ if self.config.get_libtorrent_enabled(): return self.lm.ltmgr.start_download_from_uri(uri, dconfig=download_config) raise OperationNotEnabledByConfigurationException() def start_download_from_tdef(self, torrent_definition, download_startup_config=None, hidden=False): """ Creates a Download object and adds it to the session. The passed ContentDef and DownloadStartupConfig are copied into the new Download object. The Download is then started and checkpointed. If a checkpointed version of the Download is found, that is restarted overriding the saved DownloadStartupConfig if "download_startup_config" is not None. Locking is done by LaunchManyCore. :param torrent_definition: a finalized TorrentDef :param download_startup_config: a DownloadStartupConfig or None, in which case a new DownloadStartupConfig() is created with its default settings and the result becomes the runtime config of this Download :param hidden: whether this torrent should be added to the mypreference table :return: a Download """ if self.config.get_libtorrent_enabled(): return self.lm.add(torrent_definition, download_startup_config, hidden=hidden) raise OperationNotEnabledByConfigurationException() def resume_download_from_file(self, filename): """ Recreates Download from resume file. Note: this cannot be made into a method of Download, as the Download needs to be bound to a session, it cannot exist independently. :return: a Download object :raises: a NotYetImplementedException """ raise NotYetImplementedException() def get_downloads(self): """ Returns a copy of the list of Downloads. Locking is done by LaunchManyCore. :return: a list of Download objects """ return self.lm.get_downloads() def get_download(self, infohash): """ Returns the Download object for this hash. Locking is done by LaunchManyCore. :return: a Download object """ return self.lm.get_download(infohash) def has_download(self, infohash): """ Checks if the torrent download already exists. :param infohash: The torrent infohash :return: True or False indicating if the torrent download already exists """ return self.lm.download_exists(infohash) def remove_download(self, download, remove_content=False, remove_state=True, hidden=False): """ Stops the download and removes it from the session. Note that LaunchManyCore locks. :param download: the Download to remove :param remove_content: whether to delete the already downloaded content from disk :param remove_state: whether to delete the metadata files of the downloaded content from disk :param hidden: whether this torrent is added to the mypreference table and this entry should be removed """ # locking by lm return self.lm.remove(download, removecontent=remove_content, removestate=remove_state, hidden=hidden) def remove_download_by_id(self, infohash, remove_content=False, remove_state=True): """ Remove a download by it's infohash. We can only remove content when the download object is found, otherwise only the state is removed. :param infohash: the download to remove :param remove_content: whether to delete the already downloaded content from disk :param remove_state: whether to remove the metadata files from disk """ download_list = self.get_downloads() for download in download_list: if download.get_def().get_infohash() == infohash: return self.remove_download(download, remove_content, remove_state) self.lm.remove_id(infohash) def set_download_states_callback(self, user_callback, interval=1.0): """ See Download.set_state_callback. Calls user_callback with a list of DownloadStates, one for each Download in the Session as first argument. The user_callback must return a tuple (when, getpeerlist) that indicates when to invoke the callback again (as a number of seconds from now, or < 0.0 if not at all) and whether to also include the details of the connected peers in the DownloadStates on that next call. The callback will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. :param user_callback: a function adhering to the above spec :param interval: time in between the download states callback's """ self.lm.set_download_states_callback(user_callback, interval) # # Config parameters that only exist at runtime # def get_permid(self): """ Returns the PermID of the Session, as determined by the TriblerConfig.set_permid() parameter. A PermID is a public key. :return: the PermID encoded in a string in DER format """ return str(self.keypair.pub().get_der()) # # Notification of events in the Session # def add_observer(self, observer_function, subject, change_types=None, object_id=None, cache=0): """ Add an observer function function to the Session. The observer function will be called when one of the specified events (changeTypes) occurs on the specified subject. The function will be called by a popup thread which can be used indefinitely (within reason) by the higher level code. Note that this function is called by any thread and is thread safe. :param observer_function: should accept as its first argument the subject, as second argument the changeType, as third argument an object_id (e.g. the primary key in the observed database) and an optional list of arguments. :param subject: the subject to observe, one of NTFY_* subjects (see simpledefs). :param change_types: the list of events to be notified of one of NTFY_* events. :param object_id: The specific object in the subject to monitor (e.g. a specific primary key in a database to monitor for updates.) :param cache: the time to bundle/cache events matching this function """ change_types = change_types or [NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE] self.notifier.add_observer(observer_function, subject, change_types, object_id, cache=cache) def remove_observer(self, function): """ Remove observer function. No more callbacks will be made. This function is called by any thread and is thread safe. :param function: the observer function to remove. """ self.notifier.remove_observer(function) def open_dbhandler(self, subject): """ Opens a connection to the specified database. Only the thread calling this method may use this connection. The connection must be closed with close_dbhandler() when this thread exits. This function is called by any thread. ;param subject: the database to open. Must be one of the subjects specified here. :return: a reference to a DBHandler class for the specified subject or None when the Session was not started with megacache enabled. """ if not self.config.get_megacache_enabled(): raise OperationNotEnabledByConfigurationException() if subject == NTFY_PEERS: return self.lm.peer_db elif subject == NTFY_TORRENTS: return self.lm.torrent_db elif subject == NTFY_MYPREFERENCES: return self.lm.mypref_db elif subject == NTFY_VOTECAST: return self.lm.votecast_db elif subject == NTFY_CHANNELCAST: return self.lm.channelcast_db else: raise ValueError(u"Cannot open DB subject: %s" % subject) @staticmethod def close_dbhandler(database_handler): """Closes the given database connection.""" database_handler.close() def get_tribler_statistics(self): """Return a dictionary with general Tribler statistics.""" return TriblerStatistics(self).get_tribler_statistics() def get_dispersy_statistics(self): """Return a dictionary with general Dispersy statistics.""" return TriblerStatistics(self).get_dispersy_statistics() def get_dispersy_community_statistics(self): """Return a dictionary with Dispersy communities statistics.""" return TriblerStatistics(self).get_dispersy_community_statistics() def get_ipv8_overlay_statistics(self): """Return a dictionary with IPv8 overlay statistics.""" return TriblerStatistics(self).get_ipv8_overlays_statistics() # # Persistence and shutdown # def load_checkpoint(self): """ Restart Downloads from a saved checkpoint, if any. Note that we fetch information from the user download choices since it might be that a user has stopped a download. In that case, the download should not be resumed immediately when being loaded by libtorrent. """ self.lm.load_checkpoint() def checkpoint(self): """ Saves the internal session state to the Session's state dir. Checkpoints the downloads via the LaunchManyCore instance. This function is called by any thread. """ self.lm.checkpoint_downloads() @blocking_call_on_reactor_thread def start_database(self): """ Start the SQLite database. """ db_path = os.path.join(self.config.get_state_dir(), DB_FILE_RELATIVE_PATH) self.sqlite_db = SQLiteCacheDB(db_path) self.readable_status = STATE_OPEN_DB self.sqlite_db.initialize() self.sqlite_db.initial_begin() @blocking_call_on_reactor_thread def start(self): """ Start a Tribler session by initializing the LaunchManyCore class, opening the database and running the upgrader. Returns a deferred that fires when the Tribler session is ready for use. """ # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_http_api_enabled(): self.lm.api_manager = RESTManager(self) self.readable_status = STATE_START_API self.lm.api_manager.start() self.start_database() if self.upgrader_enabled: upgrader = TriblerUpgrader(self, self.sqlite_db) self.readable_status = STATE_UPGRADING_READABLE upgrader.run() startup_deferred = self.lm.register(self, self.session_lock) def load_checkpoint(_): if self.config.get_libtorrent_enabled(): self.readable_status = STATE_LOAD_CHECKPOINTS self.load_checkpoint() self.readable_status = STATE_READABLE_STARTED return startup_deferred.addCallback(load_checkpoint) @blocking_call_on_reactor_thread def shutdown(self): """ Checkpoints the session and closes it, stopping the download engine. This method has to be called from the reactor thread. """ assert isInIOThread() @inlineCallbacks def on_early_shutdown_complete(_): """ Callback that gets called when the early shutdown has been completed. Continues the shutdown procedure that is dependant on the early shutdown. :param _: ignored parameter of the Deferred """ self.config.write() yield self.checkpoint_downloads() self.lm.shutdown_downloads() self.lm.network_shutdown() if self.sqlite_db: self.sqlite_db.close() self.sqlite_db = None return self.lm.early_shutdown().addCallback(on_early_shutdown_complete) def has_shutdown(self): """ Whether the Session has completely shutdown, i.e., its internal threads are finished and it is safe to quit the process the Session is running in. :return: a boolean. """ return self.lm.sessdoneflag.isSet() def get_downloads_pstate_dir(self): """ Returns the directory in which to checkpoint the Downloads in this Session. This function is called by the network thread. """ return os.path.join(self.config.get_state_dir(), STATEDIR_DLPSTATE_DIR) def download_torrentfile(self, infohash=None, user_callback=None, priority=0): """ Try to download the torrent file without a known source. A possible source could be the DHT. If the torrent is received successfully, the user_callback method is called with the infohash as first and the contents of the torrent file (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. :param infohash: the infohash of the torrent :param user_callback: a function adhering to the above spec :param priority: the priority of this download """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(None, infohash, user_callback=user_callback, priority=priority) def download_torrentfile_from_peer(self, candidate, infohash=None, user_callback=None, priority=0): """ Ask the designated peer to send us the torrent file for the torrent identified by the passed infohash. If the torrent is successfully received, the user_callback method is called with the infohash as first and the contents of the torrent file (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. :param candidate: the designated peer :param infohash: the infohash of the torrent :param user_callback: a function adhering to the above spec :param priority: priority of this request """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrent(candidate, infohash, user_callback=user_callback, priority=priority) def download_torrentmessage_from_peer(self, candidate, infohash, user_callback, priority=0): """ Ask the designated peer to send us the torrent message for the torrent identified by the passed infohash. If the torrent message is successfully received, the user_callback method is called with the infohash as first and the contents of the torrent file (bencoded dict) as second parameter. If the torrent could not be obtained, the callback is not called. The torrent will have been added to the TorrentDBHandler (if enabled) at the time of the call. :param candidate: the designated peer :param infohash: the infohash of the torrent :param user_callback: a function adhering to the above spec :param priority: priority of this request """ if not self.lm.rtorrent_handler: raise OperationNotEnabledByConfigurationException() self.lm.rtorrent_handler.download_torrentmessage(candidate, infohash, user_callback, priority) def get_dispersy_instance(self): if not self.config.get_dispersy_enabled(): raise OperationNotEnabledByConfigurationException() return self.lm.dispersy def get_ipv8_instance(self): if not self.config.get_ipv8_enabled(): raise OperationNotEnabledByConfigurationException() return self.lm.ipv8 def get_libtorrent_process(self): if not self.config.get_libtorrent_enabled(): raise OperationNotEnabledByConfigurationException() return self.lm.ltmgr # # Internal persistence methods # def checkpoint_downloads(self): """Checkpoints the downloads.""" return self.lm.checkpoint_downloads() def update_trackers(self, infohash, trackers): """ Updates the trackers of a torrent. :param infohash: infohash of the torrent that needs to be updated :param trackers: A list of tracker urls """ return self.lm.update_trackers(infohash, trackers) def has_collected_torrent(self, infohash): """ Checks if the given torrent infohash exists in the torrent_store database. :param infohash: The given infohash binary :return: True or False indicating if we have the torrent """ if not self.config.get_torrent_store_enabled(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") return hexlify(infohash) in self.lm.torrent_store def get_collected_torrent(self, infohash): """ Gets the given torrent from the torrent_store database. :param infohash: the given infohash binary :return: the torrent data if exists, None otherwise """ if not self.config.get_torrent_store_enabled(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") return self.lm.torrent_store.get(hexlify(infohash)) def save_collected_torrent(self, infohash, data): """ Saves the given torrent into the torrent_store database. :param infohash: the given infohash binary :param data: the torrent file data """ if not self.config.get_torrent_store_enabled(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") self.lm.torrent_store.put(hexlify(infohash), data) def delete_collected_torrent(self, infohash): """ Deletes the given torrent from the torrent_store database. :param infohash: the given infohash binary """ if not self.config.get_torrent_store_enabled(): raise OperationNotEnabledByConfigurationException("torrent_store is not enabled") del self.lm.torrent_store[hexlify(infohash)] def search_remote_torrents(self, keywords): """ Searches for remote torrents through SearchCommunity with the given keywords. :param keywords: the given keywords :return: the number of requests made """ if not self.config.get_torrent_search_enabled(): raise OperationNotEnabledByConfigurationException("torrent_search is not enabled") return self.lm.search_manager.search_for_torrents(keywords) def search_remote_channels(self, keywords): """ Searches for remote channels through AllChannelCommunity with the given keywords. :param keywords: the given keywords """ if not self.config.get_channel_search_enabled(): raise OperationNotEnabledByConfigurationException("channel_search is not enabled") self.lm.search_manager.search_for_channels(keywords) @staticmethod def create_torrent_file(file_path_list, params=None): """ Creates a torrent file. :param file_path_list: files to add in torrent file :param params: optional parameters for torrent file :return: a Deferred that fires when the torrent file has been created """ params = params or {} return threads.deferToThread(torrent_utils.create_torrent_file, file_path_list, params) def create_channel(self, name, description, mode=u'closed'): """ Creates a new Channel. :param name: name of the Channel :param description: description of the Channel :param mode: mode of the Channel ('open', 'semi-open', or 'closed') :return: a channel ID :raises a DuplicateChannelNameError if name already exists """ return self.lm.channel_manager.create_channel(name, description, mode) def add_torrent_def_to_channel(self, channel_id, torrent_def, extra_info={}, forward=True): """ Adds a TorrentDef to a Channel. :param channel_id: id of the Channel to add the Torrent to :param torrent_def: definition of the Torrent to add :param extra_info: description of the Torrent to add :param forward: when True the messages are forwarded (as defined by their message destination policy) to other nodes in the community. This parameter should (almost always) be True, its inclusion is mostly to allow certain debugging scenarios """ # Make sure that this new torrent_def is also in collected torrents self.lm.rtorrent_handler.save_torrent(torrent_def) channelcast_db = self.open_dbhandler(NTFY_CHANNELCAST) if channelcast_db.hasTorrent(channel_id, torrent_def.infohash): raise DuplicateTorrentFileError("This torrent file already exists in your channel.") dispersy_cid = str(channelcast_db.getDispersyCIDFromChannelId(channel_id)) community = self.get_dispersy_instance().get_community(dispersy_cid) community._disp_create_torrent( torrent_def.infohash, long(time.time()), torrent_def.get_name_as_unicode(), tuple(torrent_def.get_files_with_length()), torrent_def.get_trackers_as_single_tuple(), forward=forward) if 'description' in extra_info: desc = extra_info['description'].strip() if desc != '': data = channelcast_db.getTorrentFromChannelId(channel_id, torrent_def.infohash, ['ChannelTorrents.id']) community.modifyTorrent(data, {'description': desc}, forward=forward) def check_torrent_health(self, infohash, timeout=20, scrape_now=False): """ Checks the given torrent's health on its trackers. :param infohash: the given torrent infohash :param timeout: time to wait while performing the request :param scrape_now: flag to scrape immediately """ if self.lm.torrent_checker: return self.lm.torrent_checker.add_gui_request(infohash, timeout=timeout, scrape_now=scrape_now) return fail(Failure(RuntimeError("Torrent checker not available"))) def get_thumbnail_data(self, thumb_hash): """ Gets the thumbnail data. :param thumb_hash: the thumbnail SHA1 hash :return: the thumbnail data """ if not self.lm.metadata_store: raise OperationNotEnabledByConfigurationException("libtorrent is not enabled") return self.lm.rtorrent_handler.get_metadata(thumb_hash)
def test_failed_commit(self): sqlite_test_2 = SQLiteCacheDB( os.path.join(self.session_base_dir, "test_db.db"), self.tribler_db_script) sqlite_test_2.initialize() sqlite_test_2.write_version(4)
class TestContentRepositoryWithRealDatabase(TestBase): """ Tests content repository with real database. """ def setUp(self): super(TestContentRepositoryWithRealDatabase, self).setUp() session_base_dir = self.temporary_directory() tar = tarfile.open( os.path.join(TESTS_DATA_DIR, 'bak_new_tribler.sdb.tar.gz'), 'r|gz') tar.extractall(session_base_dir) db_path = os.path.join(session_base_dir, 'bak_new_tribler.sdb') self.sqlitedb = SQLiteCacheDB(db_path, busytimeout=BUSYTIMEOUT) self.sqlitedb.initialize() session = MockObject() session.sqlite_db = self.sqlitedb session.notifier = MockObject() self.torrent_db = TorrentDBHandler(session) channel_db = MockObject() self.content_repository = ContentRepository(self.torrent_db, channel_db) def tearDown(self): self.torrent_db.close() self.sqlitedb.close() super(TestContentRepositoryWithRealDatabase, self).tearDown() def test_update_db_from_search_results(self): """ Test if database is properly updated with the search results. Should not raise any UnicodeDecodeError. """ # Add a torrent infohash before updating from search results infohash = unhexlify('ed81da94d21ad1b305133f2726cdaec5a57fed98') self.content_repository.torrent_db.addOrGetTorrentID(infohash) # Sample search results name = 'Puppy.Linux.manual.301.espa\xc3\xb1ol.pdf' length = random.randint(1000, 9999) num_files = random.randint(1, 10) category_list = ['other'] creation_date = random.randint(1000000, 111111111) seeders = random.randint(10, 200) leechers = random.randint(5, 1000) cid = None search_results = [[ infohash, name, length, num_files, category_list, creation_date, seeders, leechers, cid ]] # Update from search results self.content_repository.update_from_torrent_search_results( search_results) # Check if database has correct results torrent_info = self.content_repository.get_torrent(infohash) expected_name = u'Puppy.Linux.manual.301.espa\xc3\xb1ol.pdf' self.assertEqual(expected_name, torrent_info['name']) self.assertEqual(seeders, torrent_info['num_seeders']) self.assertEqual(leechers, torrent_info['num_leechers']) self.assertEqual(creation_date, torrent_info['creation_date']) self.assertEqual(num_files, torrent_info['num_files']) self.assertEqual(length, torrent_info['length'])