class ZStormResourceManager(TestResourceManager): """Provide a L{ZStorm} resource to be used in test cases. The constructor is passed the details of the L{Store}s to be registered in the provided L{ZStore} resource. Then the C{make} and C{clean} methods make sure that such L{Store}s are properly setup and cleaned for each test. @param databases: A C{list} of C{dict}s holding the following keys: - 'name', the name of the store to be registered. - 'uri', the database URI to use to create the store. - 'schema', optionally, the L{Schema} for the tables in the store, if not given no schema will be applied. - 'schema-uri', optionally an alternate URI to use for applying the schema, if not given it defaults to 'uri'. @ivar force_delete: If C{True} for running L{Schema.delete} on a L{Store} even if no commit was performed by the test. Useful when running a test in a subprocess that might commit behind our back. @ivar use_global_zstorm: If C{True} then the C{global_zstorm} object from C{storm.zope.zstorm} will be used, instead of creating a new one. This is useful for code loading the zcml directives of C{storm.zope}. @ivar schema_stamp_dir: Optionally, a path to a directory that will be used to save timestamps of the schema's patch packages, so schema upgrades will be performed only when needed. This is just an optimisation to let the resource setup a bit faster. @ivar vertical_patching: If C{True}, patches will be applied "vertically", meaning that all patches for the first store will be applied, then all patches for the second store etc. Otherwise, if set to C{False} patches will be applied "horizontally" (see L{Sharding.upgrade}). The default is C{True} just because of backward-compatibility, but normally you should set it to C{False}. """ force_delete = False use_global_zstorm = False schema_stamp_dir = None vertical_patching = True def __init__(self, databases): super(ZStormResourceManager, self).__init__() self._databases = databases self._zstorm = None self._schema_zstorm = None self._commits = {} self._schemas = {} self._sharding = [] def make(self, dependencies): """Create a L{ZStorm} resource to be used by tests. @return: A L{ZStorm} object that will be shared among all tests using this resource manager. """ if self._zstorm is None: if self.use_global_zstorm: self._zstorm = global_zstorm else: self._zstorm = ZStorm() self._schema_zstorm = ZStorm() databases = self._databases # Adapt the old databases format to the new one, for backward # compatibility. This should be eventually dropped. if isinstance(databases, dict): databases = [{ "name": name, "uri": uri, "schema": schema } for name, (uri, schema) in databases.items()] # Provide the global IZStorm utility before applying patches, so # patch code can get the ztorm object if needed (e.g. looking up # other stores). provideUtility(self._zstorm) self._set_create_hook() enforce_schema = False for database in databases: name = database["name"] uri = database["uri"] schema = database.get("schema") schema_uri = database.get("schema-uri", uri) self._zstorm.set_default_uri(name, uri) if schema is not None: # The configuration for this database does not include a # schema definition, so we just setup the store (the user # code should apply the schema elsewhere, if any) self._schemas[name] = schema self._schema_zstorm.set_default_uri(name, schema_uri) schema.autocommit(False) store = self._schema_zstorm.get(name) if not self._sharding or self.vertical_patching: self._sharding.append(Sharding()) sharding = self._sharding[-1] sharding.add(store, schema) if self._has_patch_package_changed(name, schema): enforce_schema = True if enforce_schema: for sharding in self._sharding: try: sharding.upgrade() except UnknownPatchError: sharding.drop() sharding.create() except: # An unknown error occured, let's drop all timestamps # so subsequent runs won't assume that everything is # fine self._purge_schema_stamp_dir() raise else: sharding.delete() # Commit all schema changes across all stores transaction.commit() elif getUtility(IZStorm) is not self._zstorm: # This probably means that the test code has overwritten our # utility, let's re-register it. provideUtility(self._zstorm) return self._zstorm def _set_create_hook(self): """ Set a hook in ZStorm.create, so we can lazily set commit proxies. """ self._zstorm.__real_create__ = self._zstorm.create def create_hook(name, uri=None): store = self._zstorm.__real_create__(name, uri=uri) if self._schemas.get(name) is not None: # Only set commit proxies for databases that have a schema # that we can use for cleanup self._set_commit_proxy(store) return store self._zstorm.create = create_hook def _set_commit_proxy(self, store): """Set a commit proxy to keep track of commits and clean up the tables. @param store: The L{Store} to set the commit proxy on. Any commit on this store will result in the associated tables to be cleaned upon tear down. """ store.__real_commit__ = store.commit def commit_proxy(): self._commits[store] = True store.__real_commit__() store.commit = commit_proxy def _has_patch_package_changed(self, name, schema): """Whether the schema for the given database is up-to-date. As an optimisation, if the C{schema_stamp_dir} attribute is set, then this method performs a fast check based on the patch directory timestamp rather than the database patch table, so connections and upgrade queries can be skipped if there's no need. @param name: The name of the database to check. @param schema: The schema to be ensured. @return: C{True} if the patch directory has changed and the schema needs to be updated, C{False} otherwise. """ # If a schema stamp directory is set, then figure out whether there's # need to upgrade the schema by looking at timestamps. if self.schema_stamp_dir is not None: schema_mtime = self._get_schema_mtime(schema) schema_stamp_mtime = self._get_schema_stamp_mtime(name) # The modification time of the schema's patch directory matches our # timestamp, so the schema is already up-to-date if schema_mtime == schema_stamp_mtime: return False # Save the modification time of the schema's patch directory so in # subsequent runs we'll know if we're already up-to-date self._set_schema_stamp_mtime(name, schema_mtime) return True def _get_schema_mtime(self, schema): """ Return the modification time of the C{schema}'s patch directory. """ patch_directory = os.path.dirname(schema._patch_set._package.__file__) schema_stat = os.stat(patch_directory) return int(schema_stat.st_mtime) def _get_schema_stamp_mtime(self, name): """ Return the modification time of schemas's patch directory, as saved in the stamp directory. """ # Let's create the stamp directory if it doesn't exist if not os.path.exists(self.schema_stamp_dir): os.makedirs(self.schema_stamp_dir) schema_stamp_path = os.path.join(self.schema_stamp_dir, name) # Get the last schema modification time we ran the upgrade for, or -1 # if this is our first run if os.path.exists(schema_stamp_path): with open(schema_stamp_path) as fd: schema_stamp_mtime = int(fd.read()) else: schema_stamp_mtime = -1 return schema_stamp_mtime def _set_schema_stamp_mtime(self, name, schema_mtime): """ Save the schema's modification time in the stamp directory. """ schema_stamp_path = os.path.join(self.schema_stamp_dir, name) with open(schema_stamp_path, "w") as fd: fd.write("%d" % schema_mtime) def _purge_schema_stamp_dir(self): """Remove the stamp directory.""" if self.schema_stamp_dir and os.path.exists(self.schema_stamp_dir): shutil.rmtree(self.schema_stamp_dir) def clean(self, resource): """Clean up the stores after a test.""" try: for name, store in self._zstorm.iterstores(): # Ensure that the store is in a consistent state store.flush() # Clear the alive cache *before* abort is called, # to prevent a useless loop in Store.invalidate # over the alive objects store._alive.clear() finally: transaction.abort() # Clean up tables after each test if a commit was made needs_commit = False for name, store in self._zstorm.iterstores(): if self.force_delete or store in self._commits: schema_store = self._schema_zstorm.get(name) schema = self._schemas[name] schema.delete(schema_store) needs_commit = True if needs_commit: transaction.commit() self._commits = {}
class ZStormTest(TestHelper): def is_supported(self): return has_transaction def setUp(self): self.zstorm = ZStorm() def tearDown(self): # Reset the utility to cleanup the StoreSynchronizer's from the # transaction. self.zstorm._reset() # Free the transaction to avoid having errors that cross # test cases. # XXX cjwatson 2019-05-29: transaction 2.4.0 changed # ThreadTransactionManager to wrap TransactionManager rather than # inheriting from it. For now, cope with either. Simplify this # once transaction 2.4.0 is old enough that we can reasonably just # test-depend on it. manager = transaction.manager if isinstance(manager, ThreadTransactionManager): try: manager.free except AttributeError: # transaction >= 2.4.0 manager = manager.manager manager.free(transaction.get()) def test_create(self): store = self.zstorm.create(None, "sqlite:") self.assertTrue(isinstance(store, Store)) def test_create_twice_unnamed(self): store = self.zstorm.create(None, "sqlite:") store.execute("CREATE TABLE test (id INTEGER)") store.commit() store = self.zstorm.create(None, "sqlite:") self.assertRaises(OperationalError, store.execute, "SELECT * FROM test") def test_create_twice_same_name(self): store = self.zstorm.create("name", "sqlite:") self.assertRaises(ZStormError, self.zstorm.create, "name", "sqlite:") def test_create_and_get_named(self): store = self.zstorm.create("name", "sqlite:") self.assertTrue(self.zstorm.get("name") is store) def test_create_and_get_named_another_thread(self): store = self.zstorm.create("name", "sqlite:") raised = [] def f(): try: self.zstorm.get("name") except ZStormError: raised.append(True) thread = threading.Thread(target=f) thread.start() thread.join() self.assertTrue(raised) def test_get_unexistent(self): self.assertRaises(ZStormError, self.zstorm.get, "name") def test_get_with_uri(self): store = self.zstorm.get("name", "sqlite:") self.assertTrue(isinstance(store, Store)) self.assertTrue(self.zstorm.get("name") is store) self.assertTrue(self.zstorm.get("name", "sqlite:") is store) def test_set_default_uri(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.get("name") self.assertTrue(isinstance(store, Store)) def test_create_default(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.create("name") self.assertTrue(isinstance(store, Store)) def test_create_default_twice(self): self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.create("name") self.assertRaises(ZStormError, self.zstorm.create, "name") def test_iterstores(self): store1 = self.zstorm.create(None, "sqlite:") store2 = self.zstorm.create(None, "sqlite:") store3 = self.zstorm.create("name", "sqlite:") stores = [] for name, store in self.zstorm.iterstores(): stores.append((name, store)) self.assertEqual(len(stores), 3) self.assertEqual(set(stores), set([(None, store1), (None, store2), ("name", store3)])) def test_get_name(self): store = self.zstorm.create("name", "sqlite:") self.assertEqual(self.zstorm.get_name(store), "name") def test_get_name_with_removed_store(self): store = self.zstorm.create("name", "sqlite:") self.assertEqual(self.zstorm.get_name(store), "name") self.zstorm.remove(store) self.assertEqual(self.zstorm.get_name(store), None) def test_default_databases(self): self.zstorm.set_default_uri("name1", "sqlite:1") self.zstorm.set_default_uri("name2", "sqlite:2") self.zstorm.set_default_uri("name3", "sqlite:3") default_uris = self.zstorm.get_default_uris() self.assertEqual(default_uris, {"name1": "sqlite:1", "name2": "sqlite:2", "name3": "sqlite:3"}) def test_register_store_for_tpc_transaction(self): """ Setting a store to use two-phase-commit mode, makes ZStorm call its begin() method when it joins the transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") xids = [] store.begin = lambda xid: xids.append(xid) store.execute("SELECT 1") [xid] = xids self.assertEqual(0, xid.format_id) self.assertEqual("_storm", xid.global_transaction_id[:6]) self.assertEqual("name", xid.branch_qualifier) def test_register_store_for_tpc_transaction_uses_per_transaction_id(self): """ Two stores in two-phase-commit mode joining the same transaction share the same global transaction ID. """ self.zstorm.set_default_uri("name1", "sqlite:///%s" % self.makeFile()) self.zstorm.set_default_uri("name2", "sqlite:///%s" % self.makeFile()) self.zstorm.set_default_tpc("name1", True) self.zstorm.set_default_tpc("name2", True) store1 = self.zstorm.get("name1") store2 = self.zstorm.get("name2") xids = [] store1.begin = lambda xid: xids.append(xid) store2.begin = lambda xid: xids.append(xid) store1.execute("SELECT 1") store2.execute("SELECT 1") [xid1, xid2] = xids self.assertEqual(xid1.global_transaction_id, xid2.global_transaction_id) def test_register_store_for_tpc_transaction_uses_unique_global_ids(self): """ Each global transaction gets assigned a unique ID. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") xids = [] store.begin = lambda xid: xids.append(xid) store.execute("SELECT 1") transaction.abort() store.execute("SELECT 1") transaction.abort() [xid1, xid2] = xids self.assertNotEqual(xid1.global_transaction_id, xid2.global_transaction_id) def test_transaction_with_two_phase_commit(self): """ If a store is set to use TPC, than the associated data manager will call its prepare() and commit() methods when committing. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") calls = [] store.begin = lambda xid: calls.append("begin") store.prepare = lambda: calls.append("prepare") store.commit = lambda: calls.append("commit") store.execute("SELECT 1") transaction.commit() self.assertEqual(["begin", "prepare", "commit"], calls) def test_transaction_with_single_and_two_phase_commit_stores(self): """ When there are both stores in single-phase and two-phase mode, the ones in single-phase mode are committed first. This makes it possible to actually achieve two-phase commit behavior when only one store doesn't support TPC. """ self.zstorm.set_default_uri("name1", "sqlite:///%s" % self.makeFile()) self.zstorm.set_default_uri("name2", "sqlite:///%s" % self.makeFile()) self.zstorm.set_default_tpc("name1", True) self.zstorm.set_default_tpc("name2", False) store1 = self.zstorm.get("name1") store2 = self.zstorm.get("name2") commits = [] store1.begin = lambda xid: None store1.prepare = lambda: None store1.commit = lambda: commits.append("commit1") store2.commit = lambda: commits.append("commit2") store1.execute("SELECT 1") store2.execute("SELECT 1") transaction.commit() self.assertEqual(["commit2", "commit1"], commits) def _isInTransaction(self, store): """Check if a Store is part of the current transaction.""" for dm in transaction.get()._resources: if isinstance(dm, StoreDataManager) and dm._store is store: return True return False def assertInTransaction(self, store): """Check that the given store is joined to the transaction.""" self.assertTrue(self._isInTransaction(store), "%r should be joined to the transaction" % store) def assertNotInTransaction(self, store): """Check that the given store is not joined to the transaction.""" self.assertTrue(not self._isInTransaction(store), "%r should not be joined to the transaction" % store) def test_wb_store_joins_transaction_on_register_event(self): """The Store joins the transaction when register-transaction is emitted. The Store tests check the various operations that trigger this event. """ store = self.zstorm.get("name", "sqlite:") self.assertNotInTransaction(store) store._event.emit("register-transaction") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_commit(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.commit() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_abort(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.abort() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_tpc_commit(self): """ A store used after a two-phase commit re-joins the new transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") store.begin = lambda xid: None store.prepare = lambda: None store.commit = lambda: None store.execute("SELECT 1") transaction.commit() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_tpc_abort(self): """ A store used after a rollback during a two-phase commit re-joins the new transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") store.begin = lambda xid: None store.prepare = lambda: None store.rollback = lambda: None store.execute("SELECT 1") transaction.abort() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_remove(self): removed_store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(removed_store) for name, store in self.zstorm.iterstores(): self.assertNotEqual(store, removed_store) self.assertRaises(ZStormError, self.zstorm.get, "name") def test_wb_removed_store_does_not_join_transaction(self): """If a store has been removed, it will not join the transaction.""" store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(store) store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_removed_store_does_not_join_future_transactions(self): """If a store has been removed after joining a transaction, it will not join new transactions.""" store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") self.zstorm.remove(store) self.assertInTransaction(store) transaction.abort() store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_cross_thread_store_does_not_join_transaction(self): """If a zstorm registered thread crosses over to another thread, it will not be usable.""" store = self.zstorm.get("name", "sqlite:") failures = [] def f(): # We perform this twice to show that ZStormError is raised # consistently (i.e. not just the first time). for i in range(2): try: store.execute("SELECT 1") except ZStormError: failures.append("ZStormError raised") except Exception as exc: failures.append("Expected ZStormError, got %r" % exc) else: failures.append("Expected ZStormError, nothing raised") if self._isInTransaction(store): failures.append("store was joined to transaction") thread = threading.Thread(target=f) thread.start() thread.join() self.assertEqual(failures, ["ZStormError raised"] * 2) def test_wb_reset(self): """_reset is used to reset the zstorm utility between zope test runs. """ store = self.zstorm.get("name", "sqlite:") self.zstorm._reset() self.assertEqual(list(self.zstorm.iterstores()), []) def test_store_strong_reference(self): """ The zstorm utility should be a strong reference to named stores so that it doesn't recreate stores uselessly. """ store = self.zstorm.get("name", "sqlite:") store_ref = weakref.ref(store) transaction.abort() del store gc.collect() self.assertNotIdentical(store_ref(), None) store = self.zstorm.get("name") self.assertIdentical(store_ref(), store)
class ZStormTest(TestHelper): def is_supported(self): return has_transaction def setUp(self): self.zstorm = ZStorm() def tearDown(self): # Reset the utility to cleanup the StoreSynchronizer's from the # transaction. self.zstorm._reset() # Free the transaction to avoid having errors that cross # test cases. transaction.manager.free(transaction.get()) def test_create(self): store = self.zstorm.create(None, "sqlite:") self.assertTrue(isinstance(store, Store)) def test_create_twice_unnamed(self): store = self.zstorm.create(None, "sqlite:") store.execute("CREATE TABLE test (id INTEGER)") store.commit() store = self.zstorm.create(None, "sqlite:") self.assertRaises(OperationalError, store.execute, "SELECT * FROM test") def test_create_twice_same_name(self): store = self.zstorm.create("name", "sqlite:") self.assertRaises(ZStormError, self.zstorm.create, "name", "sqlite:") def test_create_and_get_named(self): store = self.zstorm.create("name", "sqlite:") self.assertTrue(self.zstorm.get("name") is store) def test_create_and_get_named_another_thread(self): store = self.zstorm.create("name", "sqlite:") raised = [] def f(): try: self.zstorm.get("name") except ZStormError: raised.append(True) thread = threading.Thread(target=f) thread.start() thread.join() self.assertTrue(raised) def test_get_unexistent(self): self.assertRaises(ZStormError, self.zstorm.get, "name") def test_get_with_uri(self): store = self.zstorm.get("name", "sqlite:") self.assertTrue(isinstance(store, Store)) self.assertTrue(self.zstorm.get("name") is store) self.assertTrue(self.zstorm.get("name", "sqlite:") is store) def test_set_default_uri(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.get("name") self.assertTrue(isinstance(store, Store)) def test_create_default(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.create("name") self.assertTrue(isinstance(store, Store)) def test_create_default_twice(self): self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.create("name") self.assertRaises(ZStormError, self.zstorm.create, "name") def test_iterstores(self): store1 = self.zstorm.create(None, "sqlite:") store2 = self.zstorm.create(None, "sqlite:") store3 = self.zstorm.create("name", "sqlite:") stores = [] for name, store in self.zstorm.iterstores(): stores.append((name, store)) self.assertEquals(len(stores), 3) self.assertEquals(set(stores), set([(None, store1), (None, store2), ("name", store3)])) def test_get_name(self): store = self.zstorm.create("name", "sqlite:") self.assertEquals(self.zstorm.get_name(store), "name") def test_get_name_with_removed_store(self): store = self.zstorm.create("name", "sqlite:") self.assertEquals(self.zstorm.get_name(store), "name") self.zstorm.remove(store) self.assertEquals(self.zstorm.get_name(store), None) def test_default_databases(self): self.zstorm.set_default_uri("name1", "sqlite:1") self.zstorm.set_default_uri("name2", "sqlite:2") self.zstorm.set_default_uri("name3", "sqlite:3") default_uris = self.zstorm.get_default_uris() self.assertEquals(default_uris, {"name1": "sqlite:1", "name2": "sqlite:2", "name3": "sqlite:3"}) def _isInTransaction(self, store): """Check if a Store is part of the current transaction.""" for dm in transaction.get()._resources: if isinstance(dm, StoreDataManager) and dm._store is store: return True return False def assertInTransaction(self, store): """Check that the given store is joined to the transaction.""" self.assertTrue(self._isInTransaction(store), "%r should be joined to the transaction" % store) def assertNotInTransaction(self, store): """Check that the given store is not joined to the transaction.""" self.assertTrue(not self._isInTransaction(store), "%r should not be joined to the transaction" % store) def test_wb_store_joins_transaction_on_register_event(self): """The Store joins the transaction when register-transaction is emitted. The Store tests check the various operations that trigger this event. """ store = self.zstorm.get("name", "sqlite:") self.assertNotInTransaction(store) store._event.emit("register-transaction") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_commit(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.commit() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_abort(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.abort() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_remove(self): removed_store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(removed_store) for name, store in self.zstorm.iterstores(): self.assertNotEquals(store, removed_store) self.assertRaises(ZStormError, self.zstorm.get, "name") def test_wb_removed_store_does_not_join_transaction(self): """If a store has been removed, it will not join the transaction.""" store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(store) store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_removed_store_does_not_join_future_transactions(self): """If a store has been removed after joining a transaction, it will not join new transactions.""" store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") self.zstorm.remove(store) self.assertInTransaction(store) transaction.abort() store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_cross_thread_store_does_not_join_transaction(self): """If a zstorm registered thread crosses over to another thread, it will not be usable.""" store = self.zstorm.get("name", "sqlite:") failures = [] def f(): # We perform this twice to show that ZStormError is raised # consistently (i.e. not just the first time). for i in range(2): try: store.execute("SELECT 1") except ZStormError: failures.append("ZStormError raised") except Exception, exc: failures.append("Expected ZStormError, got %r" % exc) else: failures.append("Expected ZStormError, nothing raised") if self._isInTransaction(store): failures.append("store was joined to transaction") thread = threading.Thread(target=f) thread.start() thread.join() self.assertEqual(failures, ["ZStormError raised"] * 2)
class ZStormTest(TestHelper): def is_supported(self): return has_transaction def setUp(self): self.zstorm = ZStorm() def tearDown(self): # Reset the utility to cleanup the StoreSynchronizer's from the # transaction. self.zstorm._reset() # Free the transaction to avoid having errors that cross # test cases. transaction.manager.free(transaction.get()) def test_create(self): store = self.zstorm.create(None, "sqlite:") self.assertTrue(isinstance(store, Store)) def test_create_twice_unnamed(self): store = self.zstorm.create(None, "sqlite:") store.execute("CREATE TABLE test (id INTEGER)") store.commit() store = self.zstorm.create(None, "sqlite:") self.assertRaises(OperationalError, store.execute, "SELECT * FROM test") def test_create_twice_same_name(self): store = self.zstorm.create("name", "sqlite:") self.assertRaises(ZStormError, self.zstorm.create, "name", "sqlite:") def test_create_and_get_named(self): store = self.zstorm.create("name", "sqlite:") self.assertTrue(self.zstorm.get("name") is store) def test_create_and_get_named_another_thread(self): store = self.zstorm.create("name", "sqlite:") raised = [] def f(): try: self.zstorm.get("name") except ZStormError: raised.append(True) thread = threading.Thread(target=f) thread.start() thread.join() self.assertTrue(raised) def test_get_unexistent(self): self.assertRaises(ZStormError, self.zstorm.get, "name") def test_get_with_uri(self): store = self.zstorm.get("name", "sqlite:") self.assertTrue(isinstance(store, Store)) self.assertTrue(self.zstorm.get("name") is store) self.assertTrue(self.zstorm.get("name", "sqlite:") is store) def test_set_default_uri(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.get("name") self.assertTrue(isinstance(store, Store)) def test_create_default(self): self.zstorm.set_default_uri("name", "sqlite:") store = self.zstorm.create("name") self.assertTrue(isinstance(store, Store)) def test_create_default_twice(self): self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.create("name") self.assertRaises(ZStormError, self.zstorm.create, "name") def test_iterstores(self): store1 = self.zstorm.create(None, "sqlite:") store2 = self.zstorm.create(None, "sqlite:") store3 = self.zstorm.create("name", "sqlite:") stores = [] for name, store in self.zstorm.iterstores(): stores.append((name, store)) self.assertEquals(len(stores), 3) self.assertEquals(set(stores), set([(None, store1), (None, store2), ("name", store3)])) def test_get_name(self): store = self.zstorm.create("name", "sqlite:") self.assertEquals(self.zstorm.get_name(store), "name") def test_get_name_with_removed_store(self): store = self.zstorm.create("name", "sqlite:") self.assertEquals(self.zstorm.get_name(store), "name") self.zstorm.remove(store) self.assertEquals(self.zstorm.get_name(store), None) def test_default_databases(self): self.zstorm.set_default_uri("name1", "sqlite:1") self.zstorm.set_default_uri("name2", "sqlite:2") self.zstorm.set_default_uri("name3", "sqlite:3") default_uris = self.zstorm.get_default_uris() self.assertEquals(default_uris, {"name1": "sqlite:1", "name2": "sqlite:2", "name3": "sqlite:3"}) def test_register_store_for_tpc_transaction(self): """ Setting a store to use two-phase-commit mode, makes ZStorm call its begin() method when it joins the transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") xids = [] store.begin = lambda xid: xids.append(xid) store.execute("SELECT 1") [xid] = xids self.assertEqual(0, xid.format_id) self.assertEqual("_storm", xid.global_transaction_id[:6]) self.assertEqual("name", xid.branch_qualifier) def test_register_store_for_tpc_transaction_uses_per_transaction_id(self): """ Two stores in two-phase-commit mode joining the same transaction share the same global transaction ID. """ self.zstorm.set_default_uri("name1", "sqlite:1") self.zstorm.set_default_uri("name2", "sqlite:2") self.zstorm.set_default_tpc("name1", True) self.zstorm.set_default_tpc("name2", True) store1 = self.zstorm.get("name1") store2 = self.zstorm.get("name2") xids = [] store1.begin = lambda xid: xids.append(xid) store2.begin = lambda xid: xids.append(xid) store1.execute("SELECT 1") store2.execute("SELECT 1") [xid1, xid2] = xids self.assertEqual(xid1.global_transaction_id, xid2.global_transaction_id) def test_register_store_for_tpc_transaction_uses_unique_global_ids(self): """ Each global transaction gets assigned a unique ID. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") xids = [] store.begin = lambda xid: xids.append(xid) store.execute("SELECT 1") transaction.abort() store.execute("SELECT 1") transaction.abort() [xid1, xid2] = xids self.assertNotEqual(xid1.global_transaction_id, xid2.global_transaction_id) def test_transaction_with_two_phase_commit(self): """ If a store is set to use TPC, than the associated data manager will call its prepare() and commit() methods when committing. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") calls = [] store.begin = lambda xid: calls.append("begin") store.prepare = lambda: calls.append("prepare") store.commit = lambda: calls.append("commit") store.execute("SELECT 1") transaction.commit() self.assertEqual(["begin", "prepare", "commit"], calls) def test_transaction_with_single_and_two_phase_commit_stores(self): """ When there are both stores in single-phase and two-phase mode, the ones in single-phase mode are committed first. This makes it possible to actually achieve two-phase commit behavior when only one store doesn't support TPC. """ self.zstorm.set_default_uri("name1", "sqlite:1") self.zstorm.set_default_uri("name2", "sqlite:2") self.zstorm.set_default_tpc("name1", True) self.zstorm.set_default_tpc("name2", False) store1 = self.zstorm.get("name1") store2 = self.zstorm.get("name2") commits = [] store1.begin = lambda xid: None store1.prepare = lambda: None store1.commit = lambda: commits.append("commit1") store2.commit = lambda: commits.append("commit2") store1.execute("SELECT 1") store2.execute("SELECT 1") transaction.commit() self.assertEqual(["commit2", "commit1"], commits) def _isInTransaction(self, store): """Check if a Store is part of the current transaction.""" for dm in transaction.get()._resources: if isinstance(dm, StoreDataManager) and dm._store is store: return True return False def assertInTransaction(self, store): """Check that the given store is joined to the transaction.""" self.assertTrue(self._isInTransaction(store), "%r should be joined to the transaction" % store) def assertNotInTransaction(self, store): """Check that the given store is not joined to the transaction.""" self.assertTrue(not self._isInTransaction(store), "%r should not be joined to the transaction" % store) def test_wb_store_joins_transaction_on_register_event(self): """The Store joins the transaction when register-transaction is emitted. The Store tests check the various operations that trigger this event. """ store = self.zstorm.get("name", "sqlite:") self.assertNotInTransaction(store) store._event.emit("register-transaction") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_commit(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.commit() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_abort(self): store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") transaction.abort() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_tpc_commit(self): """ A store used after a two-phase commit re-joins the new transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") store.begin = lambda xid: None store.prepare = lambda: None store.commit = lambda: None store.execute("SELECT 1") transaction.commit() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_wb_store_joins_transaction_on_use_after_tpc_abort(self): """ A store used after a rollback during a two-phase commit re-joins the new transaction. """ self.zstorm.set_default_uri("name", "sqlite:") self.zstorm.set_default_tpc("name", True) store = self.zstorm.get("name") store.begin = lambda xid: None store.prepare = lambda: None store.rollback = lambda: None store.execute("SELECT 1") transaction.abort() self.assertNotInTransaction(store) store.execute("SELECT 1") self.assertInTransaction(store) def test_remove(self): removed_store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(removed_store) for name, store in self.zstorm.iterstores(): self.assertNotEquals(store, removed_store) self.assertRaises(ZStormError, self.zstorm.get, "name") def test_wb_removed_store_does_not_join_transaction(self): """If a store has been removed, it will not join the transaction.""" store = self.zstorm.get("name", "sqlite:") self.zstorm.remove(store) store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_removed_store_does_not_join_future_transactions(self): """If a store has been removed after joining a transaction, it will not join new transactions.""" store = self.zstorm.get("name", "sqlite:") store.execute("SELECT 1") self.zstorm.remove(store) self.assertInTransaction(store) transaction.abort() store.execute("SELECT 1") self.assertNotInTransaction(store) def test_wb_cross_thread_store_does_not_join_transaction(self): """If a zstorm registered thread crosses over to another thread, it will not be usable.""" store = self.zstorm.get("name", "sqlite:") failures = [] def f(): # We perform this twice to show that ZStormError is raised # consistently (i.e. not just the first time). for i in range(2): try: store.execute("SELECT 1") except ZStormError: failures.append("ZStormError raised") except Exception, exc: failures.append("Expected ZStormError, got %r" % exc) else: failures.append("Expected ZStormError, nothing raised") if self._isInTransaction(store): failures.append("store was joined to transaction") thread = threading.Thread(target=f) thread.start() thread.join() self.assertEqual(failures, ["ZStormError raised"] * 2)
class ZStormResourceManager(TestResourceManager): """Provide a L{ZStorm} resource to be used in test cases. The constructor is passed the details of the L{Store}s to be registered in the provided L{ZStore} resource. Then the C{make} and C{clean} methods make sure that such L{Store}s are properly setup and cleaned for each test. @param databases: A C{list} of C{dict}s holding the following keys: - 'name', the name of the store to be registered. - 'uri', the database URI to use to create the store. - 'schema', optionally, the L{Schema} for the tables in the store, if not given no schema will be applied. - 'schema-uri', optionally an alternate URI to use for applying the schema, if not given it defaults to 'uri'. @ivar force_delete: If C{True} for running L{Schema.delete} on a L{Store} even if no commit was performed by the test. Useful when running a test in a subprocess that might commit behind our back. @ivar use_global_zstorm: If C{True} then the C{global_zstorm} object from C{storm.zope.zstorm} will be used, instead of creating a new one. This is useful for code loading the zcml directives of C{storm.zope}. @ivar schema_stamp_dir: Optionally, a path to a directory that will be used to save timestamps of the schema's patch packages, so schema upgrades will be performed only when needed. This is just an optimisation to let the resource setup a bit faster. """ force_delete = False use_global_zstorm = False schema_stamp_dir = None def __init__(self, databases): super(ZStormResourceManager, self).__init__() self._databases = databases self._zstorm = None self._schema_zstorm = None self._commits = {} self._schemas = {} def make(self, dependencies): """Create a L{ZStorm} resource to be used by tests. @return: A L{ZStorm} object that will be shared among all tests using this resource manager. """ if self._zstorm is None: if self.use_global_zstorm: self._zstorm = global_zstorm else: self._zstorm = ZStorm() self._schema_zstorm = ZStorm() databases = self._databases # Adapt the old databases format to the new one, for backward # compatibility. This should be eventually dropped. if isinstance(databases, dict): databases = [{"name": name, "uri": uri, "schema": schema} for name, (uri, schema) in databases.iteritems()] # Provide the global IZStorm utility before applying patches, so # patch code can get the ztorm object if needed (e.g. looking up # other stores). provideUtility(self._zstorm) self._set_create_hook() for database in databases: name = database["name"] uri = database["uri"] schema = database.get("schema") schema_uri = database.get("schema-uri", uri) self._zstorm.set_default_uri(name, uri) if schema is not None: # The configuration for this database does not include a # schema definition, so we just setup the store (the user # code should apply the schema elsewhere, if any) self._schemas[name] = schema self._schema_zstorm.set_default_uri(name, schema_uri) self._ensure_schema(name, schema) # Commit all schema changes across all stores transaction.commit() elif getUtility(IZStorm) is not self._zstorm: # This probably means that the test code has overwritten our # utility, let's re-register it. provideUtility(self._zstorm) return self._zstorm def _set_create_hook(self): """ Set a hook in ZStorm.create, so we can lazily set commit proxies. """ self._zstorm.__real_create__ = self._zstorm.create def create_hook(name, uri=None): store = self._zstorm.__real_create__(name, uri=uri) if self._schemas.get(name) is not None: # Only set commit proxies for databases that have a schema # that we can use for cleanup self._set_commit_proxy(store) return store self._zstorm.create = create_hook def _set_commit_proxy(self, store): """Set a commit proxy to keep track of commits and clean up the tables. @param store: The L{Store} to set the commit proxy on. Any commit on this store will result in the associated tables to be cleaned upon tear down. """ store.__real_commit__ = store.commit def commit_proxy(): self._commits[store] = True store.__real_commit__() store.commit = commit_proxy def _ensure_schema(self, name, schema): """Ensure that the schema for the given database is up-to-date. As an optimisation, if the C{schema_stamp_dir} attribute is set, then this method performs a fast check based on the patch directory timestamp rather than the database patch table, so connections and upgrade queries can be skipped if there's no need. @param name: The name of the database to check. @param schema: The schema to be ensured. """ # If a schema stamp directory is set, then figure out whether there's # need to upgrade the schema by looking at timestamps. if self.schema_stamp_dir is not None: schema_mtime = self._get_schema_mtime(schema) schema_stamp_mtime = self._get_schema_stamp_mtime(name) # The modification time of the schema's patch directory matches our # timestamp, so the schema is already up-to-date if schema_mtime == schema_stamp_mtime: return # Save the modification time of the schema's patch directory so in # subsequent runs we'll know if we're already up-to-date self._set_schema_stamp_mtime(name, schema_mtime) schema_store = self._schema_zstorm.get(name) # Disable schema autocommits, we will commit everything at once schema.autocommit(False) try: schema.upgrade(schema_store) except UnknownPatchError: schema.drop(schema_store) schema_store.commit() schema.upgrade(schema_store) else: # Clean up tables here to ensure that the first test run starts # with an empty db schema.delete(schema_store) def _get_schema_mtime(self, schema): """ Return the modification time of the C{schema}'s patch directory. """ schema_stat = os.stat(os.path.dirname(schema._patch_package.__file__)) return int(schema_stat.st_mtime) def _get_schema_stamp_mtime(self, name): """ Return the modification time of schemas's patch directory, as saved in the stamp directory. """ # Let's create the stamp directory if it doesn't exist if not os.path.exists(self.schema_stamp_dir): os.makedirs(self.schema_stamp_dir) schema_stamp_path = os.path.join(self.schema_stamp_dir, name) # Get the last schema modification time we ran the upgrade for, or -1 # if this is our first run if os.path.exists(schema_stamp_path): with open(schema_stamp_path) as fd: schema_stamp_mtime = int(fd.read()) else: schema_stamp_mtime = -1 return schema_stamp_mtime def _set_schema_stamp_mtime(self, name, schema_mtime): """ Save the schema's modification time in the stamp directory. """ schema_stamp_path = os.path.join(self.schema_stamp_dir, name) with open(schema_stamp_path, "w") as fd: fd.write("%d" % schema_mtime) def clean(self, resource): """Clean up the stores after a test.""" try: for name, store in self._zstorm.iterstores(): # Ensure that the store is in a consistent state store.flush() # Clear the alive cache *before* abort is called, # to prevent a useless loop in Store.invalidate # over the alive objects store._alive.clear() finally: transaction.abort() # Clean up tables after each test if a commit was made needs_commit = False for name, store in self._zstorm.iterstores(): if self.force_delete or store in self._commits: schema_store = self._schema_zstorm.get(name) schema = self._schemas[name] schema.delete(schema_store) needs_commit = True if needs_commit: transaction.commit() self._commits = {}