def checkLoadBlob(self): from ZODB.blob import Blob from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, handle_serials import transaction somedata = "a" * 10 blob = Blob() bd_fh = blob.open("w") bd_fh.write(somedata) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, "", t) r2 = self._storage.tpc_vote(t) serial = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise filename = self._storage.loadBlob(oid, serial) self.assertEquals(somedata, open(filename, "rb").read()) self.assert_(not (os.stat(filename).st_mode & stat.S_IWRITE)) self.assert_((os.stat(filename).st_mode & stat.S_IREAD))
def checkLoadBlob(self): from ZODB.blob import Blob from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \ handle_serials import transaction version = '' somedata = 'a' * 10 blob = Blob() bd_fh = blob.open('w') bd_fh.write(somedata) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t) r2 = self._storage.tpc_vote(t) serial = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise filename = self._storage.loadBlob(oid, serial) self.assertEquals(somedata, open(filename, 'rb').read()) self.assert_(not (os.stat(filename).st_mode & stat.S_IWRITE)) self.assert_((os.stat(filename).st_mode & stat.S_IREAD))
def checkStoreBlob(self): from ZODB.utils import oid_repr, tid_repr from ZODB.blob import Blob, BLOB_SUFFIX from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, handle_serials import transaction somedata = "a" * 10 blob = Blob() bd_fh = blob.open("w") bd_fh.write(somedata) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) self.assert_(os.path.exists(tfname)) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, "", t) r2 = self._storage.tpc_vote(t) revid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise self.assert_(not os.path.exists(tfname)) filename = self._storage.fshelper.getBlobFilename(oid, revid) self.assert_(os.path.exists(filename)) self.assertEqual(somedata, open(filename).read())
def checkStoreBlob(self): from ZODB.utils import oid_repr, tid_repr from ZODB.blob import Blob, BLOB_SUFFIX from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \ handle_serials import transaction somedata = 'a' * 10 blob = Blob() bd_fh = blob.open('w') bd_fh.write(somedata) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) self.assert_(os.path.exists(tfname)) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t) r2 = self._storage.tpc_vote(t) revid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise self.assert_(not os.path.exists(tfname)) filename = self._storage.fshelper.getBlobFilename(oid, revid) self.assert_(os.path.exists(filename)) self.assertEqual(somedata, open(filename).read())
def helper(tid, revid, x): data = zodb_pickle(MinPO(x)) t = transaction.Transaction() try: self._storage.tpc_begin(t, p64(tid)) r1 = self._storage.store(oid, revid, data, '', t) # Finish the transaction r2 = self._storage.tpc_vote(t) newrevid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise return newrevid
def checkSerialIsNoneForInitialRevision(self): eq = self.assertEqual oid = self._storage.new_oid() txn = transaction.Transaction() self._storage.tpc_begin(txn) # Use None for serial. Don't use _dostore() here because that coerces # serial=None to serial=ZERO. r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)), '', txn) r2 = self._storage.tpc_vote(txn) self._storage.tpc_finish(txn) newrevid = handle_serials(oid, r1, r2) data, revid = self._storage.load(oid, '') value = zodb_unpickle(data) eq(value, MinPO(11)) eq(revid, newrevid)
def checkSerialIsNoneForInitialRevision(self): eq = self.assertEqual oid = self._storage.new_oid() txn = transaction.Transaction() self._storage.tpc_begin(txn) # Use None for serial. Don't use _dostore() here because that coerces # serial=None to serial=ZERO. r1 = self._storage.store(oid, None, zodb_pickle(MinPO(11)), "", txn) r2 = self._storage.tpc_vote(txn) self._storage.tpc_finish(txn) newrevid = handle_serials(oid, r1, r2) data, revid = self._storage.load(oid, "") value = zodb_unpickle(data) eq(value, MinPO(11)) eq(revid, newrevid)
def dostore(self, i): data = zodb_pickle(MinPO((self.getName(), i))) t = transaction.Transaction() oid = self.oid() self.pause() self.storage.tpc_begin(t) self.pause() # Always create a new object, signified by None for revid r1 = self.storage.store(oid, None, data, "", t) self.pause() r2 = self.storage.tpc_vote(t) self.pause() self.storage.tpc_finish(t) self.pause() revid = handle_serials(oid, r1, r2) self.oids[oid] = revid
def dostore(self, i): data = zodb_pickle(MinPO((self.getName(), i))) t = transaction.Transaction() oid = self.oid() self.pause() self.storage.tpc_begin(t) self.pause() # Always create a new object, signified by None for revid r1 = self.storage.store(oid, None, data, '', t) self.pause() r2 = self.storage.tpc_vote(t) self.pause() self.storage.tpc_finish(t) self.pause() revid = handle_serials(oid, r1, r2) self.oids[oid] = revid
def checkRestoreAfterDoubleCommit(self): oid = self._storage.new_oid() revid = '\0'*8 data1 = zodb_pickle(MinPO(11)) data2 = zodb_pickle(MinPO(12)) # Begin the transaction t = transaction.Transaction() try: self._storage.tpc_begin(t) # Store an object self._storage.store(oid, revid, data1, '', t) # Store it again r1 = self._storage.store(oid, revid, data2, '', t) # Finish the transaction r2 = self._storage.tpc_vote(t) revid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise self._dst.copyTransactionsFrom(self._storage) self.compare(self._storage, self._dst)
def checkStoreAndLoadBlob(self): from ZODB.utils import oid_repr, tid_repr from ZODB.blob import Blob, BLOB_SUFFIX from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, handle_serials import transaction somedata_path = os.path.join(self.blob_cache_dir, "somedata") somedata = open(somedata_path, "w+b") for i in range(1000000): somedata.write("%s\n" % i) somedata.seek(0) blob = Blob() bd_fh = blob.open("w") ZODB.utils.cp(somedata, bd_fh) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) self.assert_(os.path.exists(tfname)) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, "", t) r2 = self._storage.tpc_vote(t) revid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise # The uncommitted data file should have been removed self.assert_(not os.path.exists(tfname)) def check_data(path): self.assert_(os.path.exists(path)) f = open(path, "rb") somedata.seek(0) d1 = d2 = 1 while d1 or d2: d1 = f.read(8096) d2 = somedata.read(8096) self.assertEqual(d1, d2) # The file should be in the cache ... filename = self._storage.fshelper.getBlobFilename(oid, revid) check_data(filename) # ... and on the server server_filename = os.path.join(self.blobdir, ZODB.blob.BushyLayout().getBlobFilePath(oid, revid)) self.assert_(server_filename.startswith(self.blobdir)) check_data(server_filename) # If we remove it from the cache and call loadBlob, it should # come back. We can do this in many threads. We'll instrument # the method that is used to request data from teh server to # verify that it is only called once. sendBlob_org = ZEO.ServerStub.StorageServer.sendBlob calls = [] def sendBlob(self, oid, serial): calls.append((oid, serial)) sendBlob_org(self, oid, serial) ZODB.blob.remove_committed(filename) returns = [] threads = [ threading.Thread(target=lambda: returns.append(self._storage.loadBlob(oid, revid))) for i in range(10) ] [thread.start() for thread in threads] [thread.join() for thread in threads] [self.assertEqual(r, filename) for r in returns] check_data(filename)
def checkTimeoutProvokingConflicts(self): self._storage = storage = self.openClientStorage() # Assert that the zeo cache is empty. self.assert_(not list(storage._cache.contents())) # Create the object oid = storage.new_oid() obj = MinPO(7) # We need to successfully commit an object now so we have something to # conflict about. t = Transaction() storage.tpc_begin(t) revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t) revid1b = storage.tpc_vote(t) revid1 = handle_serials(oid, revid1a, revid1b) storage.tpc_finish(t) # Now do a store, sleeping before the finish so as to cause a timeout. obj.value = 8 t = Transaction() old_connection_count = storage.connection_count_for_tests storage.tpc_begin(t) revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t) revid2b = storage.tpc_vote(t) revid2 = handle_serials(oid, revid2a, revid2b) # Now sleep long enough for the storage to time out. # This used to sleep for 3 seconds, and sometimes (but very rarely) # failed then. Now we try for a minute. It typically succeeds # on the second time thru the loop, and, since self.timeout is 1, # it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead # of 3). deadline = time.time() + 60 # wait up to a minute while time.time() < deadline: if (storage.is_connected() and (storage.connection_count_for_tests == old_connection_count) ): time.sleep(self.timeout / 1.8) else: break self.assert_( (not storage.is_connected()) or (storage.connection_count_for_tests > old_connection_count) ) storage._wait() self.assert_(storage.is_connected()) # We expect finish to fail. self.assertRaises(ClientDisconnected, storage.tpc_finish, t) storage.tpc_abort(t) # Now we think we've committed the second transaction, but we really # haven't. A third one should produce a POSKeyError on the server, # which manifests as a ConflictError on the client. obj.value = 9 t = Transaction() storage.tpc_begin(t) storage.store(oid, revid2, zodb_pickle(obj), '', t) self.assertRaises(ConflictError, storage.tpc_vote, t) # Even aborting won't help. storage.tpc_abort(t) self.assertRaises(ZODB.POSException.StorageTransactionError, storage.tpc_finish, t) # Try again. obj.value = 10 t = Transaction() storage.tpc_begin(t) storage.store(oid, revid2, zodb_pickle(obj), '', t) # Even aborting won't help. self.assertRaises(ConflictError, storage.tpc_vote, t) # Abort this one and try a transaction that should succeed. storage.tpc_abort(t) # Now do a store. obj.value = 11 t = Transaction() storage.tpc_begin(t) revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t) revid2b = storage.tpc_vote(t) revid2 = handle_serials(oid, revid2a, revid2b) storage.tpc_finish(t) # Now load the object and verify that it has a value of 11. data, revid = storage.load(oid, '') self.assertEqual(zodb_unpickle(data), MinPO(11)) self.assertEqual(revid, revid2)
def checkTimeoutProvokingConflicts(self): self._storage = storage = self.openClientStorage() # Assert that the zeo cache is empty. self.assert_(not list(storage._cache.contents())) # Create the object oid = storage.new_oid() obj = MinPO(7) # We need to successfully commit an object now so we have something to # conflict about. t = Transaction() storage.tpc_begin(t) revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t) revid1b = storage.tpc_vote(t) revid1 = handle_serials(oid, revid1a, revid1b) storage.tpc_finish(t) # Now do a store, sleeping before the finish so as to cause a timeout. obj.value = 8 t = Transaction() old_connection_count = storage.connection_count_for_tests storage.tpc_begin(t) revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t) revid2b = storage.tpc_vote(t) revid2 = handle_serials(oid, revid2a, revid2b) # Now sleep long enough for the storage to time out. # This used to sleep for 3 seconds, and sometimes (but very rarely) # failed then. Now we try for a minute. It typically succeeds # on the second time thru the loop, and, since self.timeout is 1, # it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead # of 3). deadline = time.time() + 60 # wait up to a minute while time.time() < deadline: if (storage.is_connected() and (storage.connection_count_for_tests == old_connection_count)): time.sleep(self.timeout / 1.8) else: break self.assert_( (not storage.is_connected()) or (storage.connection_count_for_tests > old_connection_count)) storage._wait() self.assert_(storage.is_connected()) # We expect finish to fail. self.assertRaises(ClientDisconnected, storage.tpc_finish, t) storage.tpc_abort(t) # Now we think we've committed the second transaction, but we really # haven't. A third one should produce a POSKeyError on the server, # which manifests as a ConflictError on the client. obj.value = 9 t = Transaction() storage.tpc_begin(t) storage.store(oid, revid2, zodb_pickle(obj), '', t) self.assertRaises(ConflictError, storage.tpc_vote, t) # Even aborting won't help. storage.tpc_abort(t) self.assertRaises(ZODB.POSException.StorageTransactionError, storage.tpc_finish, t) # Try again. obj.value = 10 t = Transaction() storage.tpc_begin(t) storage.store(oid, revid2, zodb_pickle(obj), '', t) # Even aborting won't help. self.assertRaises(ConflictError, storage.tpc_vote, t) # Abort this one and try a transaction that should succeed. storage.tpc_abort(t) # Now do a store. obj.value = 11 t = Transaction() storage.tpc_begin(t) revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t) revid2b = storage.tpc_vote(t) revid2 = handle_serials(oid, revid2a, revid2b) storage.tpc_finish(t) # Now load the object and verify that it has a value of 11. data, revid = storage.load(oid, '') self.assertEqual(zodb_unpickle(data), MinPO(11)) self.assertEqual(revid, revid2)
def checkStoreAndLoadBlob(self): from ZODB.utils import oid_repr, tid_repr from ZODB.blob import Blob, BLOB_SUFFIX from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \ handle_serials import transaction somedata_path = os.path.join(self.blob_cache_dir, 'somedata') somedata = open(somedata_path, 'w+b') for i in range(1000000): somedata.write("%s\n" % i) somedata.seek(0) blob = Blob() bd_fh = blob.open('w') ZODB.utils.cp(somedata, bd_fh) bd_fh.close() tfname = bd_fh.name oid = self._storage.new_oid() data = zodb_pickle(blob) self.assert_(os.path.exists(tfname)) t = transaction.Transaction() try: self._storage.tpc_begin(t) r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t) r2 = self._storage.tpc_vote(t) revid = handle_serials(oid, r1, r2) self._storage.tpc_finish(t) except: self._storage.tpc_abort(t) raise # The uncommitted data file should have been removed self.assert_(not os.path.exists(tfname)) def check_data(path): self.assert_(os.path.exists(path)) f = open(path, 'rb') somedata.seek(0) d1 = d2 = 1 while d1 or d2: d1 = f.read(8096) d2 = somedata.read(8096) self.assertEqual(d1, d2) # The file should be in the cache ... filename = self._storage.fshelper.getBlobFilename(oid, revid) check_data(filename) # ... and on the server server_filename = filename.replace(self.blob_cache_dir, self.blobdir) self.assert_(server_filename.startswith(self.blobdir)) check_data(server_filename) # If we remove it from the cache and call loadBlob, it should # come back. We can do this in many threads. We'll instrument # the method that is used to request data from teh server to # verify that it is only called once. sendBlob_org = ZEO.ServerStub.StorageServer.sendBlob calls = [] def sendBlob(self, oid, serial): calls.append((oid, serial)) sendBlob_org(self, oid, serial) ZODB.blob.remove_committed(filename) returns = [] threads = [ threading.Thread(target=lambda: returns.append( self._storage.loadBlob(oid, revid))) for i in range(10) ] [thread.start() for thread in threads] [thread.join() for thread in threads] [self.assertEqual(r, filename) for r in returns] check_data(filename)