def db_layer():
""""Create a DemoStorage that wrapps the original storage if `nodemo`
is False.
"""
base_db = Zope2.bobo_application._stuff[0]
blob_temp = None
demo_storage = DemoStorage(base=base_db._storage)
if not IBlobStorage.providedBy(demo_storage):
from ZODB.blob import BlobStorage
from tempfile import mkdtemp
blob_temp = mkdtemp()
demo_storage = BlobStorage(blob_temp, demo_storage)
#Remove from databases the main database otherwise it will result in
#an error on creation of the DB
base_db.databases.pop(base_db.database_name, None)
# new database with the new storage
wrapper_db = ZODB.DB(storage=demo_storage,
database_name=base_db.database_name,
databases=base_db.databases)
# monkey-patch the current bobo_application to use our new database
patch_bobo_application(wrapper_db)
def cleanup():
patch_bobo_application(base_db)
if blob_temp is not None:
import shutil
shutil.rmtree(blob_temp)
return cleanup, wrapper_db
def db_layer():
# create a DemoStorage that wraps the old storage
base_db = Zope2.bobo_application._stuff[0]
demo_storage = DemoStorage(base=base_db._storage)
if not IBlobStorage.providedBy(demo_storage):
from ZODB.blob import BlobStorage
from tempfile import mkdtemp
blob_temp = mkdtemp()
demo_storage = BlobStorage(blob_temp, demo_storage)
else:
blob_temp = None
# reconstruct the mount table
database_name = base_db.database_name
new_databases = dict(base_db.databases)
del new_databases[database_name]
# new database with the new storage
wrapper_db = ZODB.DB(storage=demo_storage,
database_name=database_name,
databases=new_databases)
# monkey-patch the current bobo_application to use our new database
Zope2.bobo_application._stuff = (wrapper_db, 'Application', 'Zope-Version')
def cleanup():
Zope2.bobo_application._stuff = (base_db, 'Application', 'Zope-Version')
if blob_temp is not None:
import shutil
shutil.rmtree(blob_temp)
return cleanup, wrapper_db
def _store_objects(self, writer, transaction):
for obj in writer:
oid = obj._p_oid
serial = getattr(obj, "_p_serial", z64)
if ((serial == z64)
and ((self._savepoint_storage is None) or
(oid not in self._savepoint_storage.creating)
or self._savepoint_storage.creating[oid])):
# obj is a new object
# Because obj was added, it is now in _creating, so it
# can be removed from _added. If oid wasn't in
# adding, then we are adding it implicitly.
implicitly_adding = self._added.pop(oid, None) is None
self._creating[oid] = implicitly_adding
else:
if (oid in self._invalidated
and not hasattr(obj, '_p_resolveConflict')):
raise ConflictError(object=obj)
self._modified.append(oid)
p = writer.serialize(obj) # This calls __getstate__ of obj
if isinstance(obj, Blob):
if not IBlobStorage.providedBy(self._storage):
raise Unsupported("Storing Blobs in %s is not supported." %
repr(self._storage))
if obj.opened():
raise ValueError("Can't commit with opened blobs.")
s = self._storage.storeBlob(oid, serial, p, obj._uncommitted(),
self._version, transaction)
# we invalidate the object here in order to ensure
# that that the next attribute access of its name
# unghostify it, which will cause its blob data
# to be reattached "cleanly"
obj._p_invalidate()
else:
s = self._storage.store(oid, serial, p, self._version,
transaction)
self._cache.update_object_size_estimation(oid, len(p))
obj._p_estimated_size = len(p)
self._store_count += 1
# Put the object in the cache before handling the
# response, just in case the response contains the
# serial number for a newly created object
try:
self._cache[oid] = obj
except:
# Dang, I bet it's wrapped:
# TODO: Deprecate, then remove, this.
if hasattr(obj, 'aq_base'):
self._cache[oid] = obj.aq_base
else:
raise
self._handle_serial(s, oid)
def image_update_data(self, data, content_type=None, size=None):
if isinstance(data, unicode):
raise TypeError('Data can only be str or file-like. '
'Unicode objects are expressly forbidden.')
if size is None: size=len(data)
self.size=size
if data:
if IBlobStorage.providedBy(self._p_jar.db().storage):
self.data = ZODB.blob.Blob(data)
else:
self.data = data
else:
self.data = ''
ct, width, height = getImageInfo(data)
if ct:
content_type = ct
if width >= 0 and height >= 0:
self.width = width
self.height = height
# Now we should have the correct content type, or still None
if content_type is not None: self.content_type = content_type
self.ZCacheable_invalidate()
self.ZCacheable_set(None)
self.http__refreshEtag()
def loadBlob(self, oid, serial):
"""Return the filename where the blob file can be found.
"""
if not IBlobStorage.providedBy(self._storage):
raise Unsupported(
"Blobs are not supported by the underlying storage %r." %
self._storage)
filename = self._getCleanFilename(oid, serial)
if not os.path.exists(filename):
return self._storage.loadBlob(oid, serial)
return filename
def loadBlob(self, oid, serial):
"""Return the filename where the blob file can be found.
"""
if not IBlobStorage.providedBy(self._storage):
raise Unsupported(
"Blobs are not supported by the underlying storage %r." %
self._storage)
filename = self._getCleanFilename(oid, serial)
if not os.path.exists(filename):
return self._storage.loadBlob(oid, serial)
return filename
def __init__(self):
"""Prepares for a functional test case.
"""
transaction.abort()
storage = DemoStorage("Demo Storage")
if not IBlobStorage.providedBy(storage):
raise RuntimeError
self.db = DB(storage)
self.connection = None
def file_update_data(self, data, content_type=None, size=None):
if isinstance(data, unicode):
raise TypeError('Data can only be str or file-like. '
'Unicode objects are expressly forbidden.')
if content_type is not None: self.content_type=content_type
if size is None: size=len(data)
self.size=size
if data:
if IBlobStorage.providedBy(self._p_jar.db().storage):
self.data = ZODB.blob.Blob(data)
else:
self.data = data
else:
self.data = ''
self.ZCacheable_invalidate()
self.ZCacheable_set(None)
self.http__refreshEtag()
def exportFile(self, oid, f=None):
if f is None:
f = TemporaryFile(prefix="EXP")
elif isinstance(f, six.string_types):
f = open(f, 'w+b')
f.write(b'ZEXP')
oids = [oid]
done_oids = {}
done = done_oids.__contains__
load = self._storage.load
supports_blobs = IBlobStorage.providedBy(self._storage)
while oids:
oid = oids.pop(0)
if oid in done_oids:
continue
done_oids[oid] = True
try:
p, serial = load(oid)
except:
logger.debug("broken reference for oid %s",
repr(oid),
exc_info=True)
else:
referencesf(p, oids)
f.writelines([oid, p64(len(p)), p])
if supports_blobs:
if not isinstance(self._reader.getGhost(p), Blob):
continue # not a blob
blobfilename = self._storage.loadBlob(oid, serial)
f.write(blob_begin_marker)
f.write(p64(os.stat(blobfilename).st_size))
blobdata = open(blobfilename, "rb")
cp(blobdata, f)
blobdata.close()
f.write(export_end_marker)
return f
def exportFile(self, oid, f=None):
if f is None:
f = TemporaryFile(prefix="EXP")
elif isinstance(f, six.string_types):
f = open(f,'w+b')
f.write(b'ZEXP')
oids = [oid]
done_oids = {}
done = done_oids.__contains__
load = self._storage.load
supports_blobs = IBlobStorage.providedBy(self._storage)
while oids:
oid = oids.pop(0)
if oid in done_oids:
continue
done_oids[oid] = True
try:
p, serial = load(oid)
except:
logger.debug("broken reference for oid %s", repr(oid),
exc_info=True)
else:
referencesf(p, oids)
f.writelines([oid, p64(len(p)), p])
if supports_blobs:
if not isinstance(self._reader.getGhost(p), Blob):
continue # not a blob
blobfilename = self._storage.loadBlob(oid, serial)
f.write(blob_begin_marker)
f.write(p64(os.stat(blobfilename).st_size))
blobdata = open(blobfilename, "rb")
cp(blobdata, f)
blobdata.close()
f.write(export_end_marker)
return f
def _store_objects(self, writer, transaction):
for obj in writer:
oid = obj._p_oid
serial = getattr(obj, "_p_serial", z64)
if ((serial == z64)
and ((self._savepoint_storage is None) or
(oid not in self._savepoint_storage.creating)
or self._savepoint_storage.creating[oid])):
# obj is a new object
# Because obj was added, it is now in _creating, so it
# can be removed from _added. If oid wasn't in
# adding, then we are adding it implicitly.
implicitly_adding = self._added.pop(oid, None) is None
self._creating[oid] = implicitly_adding
else:
self._modified.append(oid)
p = writer.serialize(obj) # This calls __getstate__ of obj
if len(p) >= self.large_record_size:
warnings.warn(large_object_message % (obj.__class__, len(p)))
if isinstance(obj, Blob):
if not IBlobStorage.providedBy(self._storage):
raise Unsupported("Storing Blobs in %s is not supported." %
repr(self._storage))
if obj.opened():
raise ValueError("Can't commit with opened blobs.")
blobfilename = obj._uncommitted()
if blobfilename is None:
assert serial is not None # See _uncommitted
self._modified.pop() # not modified
continue
s = self._storage.storeBlob(oid, serial, p, blobfilename, '',
transaction)
# we invalidate the object here in order to ensure
# that that the next attribute access of its name
# unghostify it, which will cause its blob data
# to be reattached "cleanly"
obj._p_invalidate()
else:
s = self._storage.store(oid, serial, p, '', transaction)
self._store_count += 1
# Put the object in the cache before handling the
# response, just in case the response contains the
# serial number for a newly created object
try:
self._cache[oid] = obj
except:
# Dang, I bet it's wrapped:
# TODO: Deprecate, then remove, this.
if hasattr(obj, 'aq_base'):
self._cache[oid] = obj.aq_base
else:
raise
self._cache.update_object_size_estimation(oid, len(p))
obj._p_estimated_size = len(p)
# if we write an object, we don't want to check if it was read
# while current. This is a convenient choke point to do this.
self._readCurrent.pop(oid, None)
if s:
# savepoint
obj._p_changed = 0 # transition from changed to up-to-date
obj._p_serial = s
def _store_objects(self, writer, transaction):
for obj in writer:
oid = obj._p_oid
serial = getattr(obj, "_p_serial", z64)
if ((serial == z64)
and
((self._savepoint_storage is None)
or (oid not in self._savepoint_storage.creating)
or self._savepoint_storage.creating[oid]
)
):
# obj is a new object
# Because obj was added, it is now in _creating, so it
# can be removed from _added. If oid wasn't in
# adding, then we are adding it implicitly.
implicitly_adding = self._added.pop(oid, None) is None
self._creating[oid] = implicitly_adding
else:
self._modified.append(oid)
p = writer.serialize(obj) # This calls __getstate__ of obj
if len(p) >= self.large_record_size:
warnings.warn(large_object_message % (obj.__class__, len(p)))
if isinstance(obj, Blob):
if not IBlobStorage.providedBy(self._storage):
raise Unsupported(
"Storing Blobs in %s is not supported." %
repr(self._storage))
if obj.opened():
raise ValueError("Can't commit with opened blobs.")
blobfilename = obj._uncommitted()
if blobfilename is None:
assert serial is not None # See _uncommitted
self._modified.pop() # not modified
continue
s = self._storage.storeBlob(oid, serial, p, blobfilename,
'', transaction)
# we invalidate the object here in order to ensure
# that that the next attribute access of its name
# unghostify it, which will cause its blob data
# to be reattached "cleanly"
obj._p_invalidate()
else:
s = self._storage.store(oid, serial, p, '', transaction)
self._store_count += 1
# Put the object in the cache before handling the
# response, just in case the response contains the
# serial number for a newly created object
try:
self._cache[oid] = obj
except:
# Dang, I bet it's wrapped:
# TODO: Deprecate, then remove, this.
if hasattr(obj, 'aq_base'):
self._cache[oid] = obj.aq_base
else:
raise
self._cache.update_object_size_estimation(oid, len(p))
obj._p_estimated_size = len(p)
# if we write an object, we don't want to check if it was read
# while current. This is a convenient choke point to do this.
self._readCurrent.pop(oid, None)
if s:
# savepoint
obj._p_changed = 0 # transition from changed to up-to-date
obj._p_serial = s
def _read_data(self, file):
import transaction
n=1 << 16
# Make sure we have an _p_jar, even if we are a new object, by
# doing a sub-transaction commit.
transaction.savepoint(optimistic=True)
if isinstance(file, str):
size=len(file)
if size<n or IBlobStorage.providedBy(self._p_jar.db().storage):
#for blobs we don't have to cut anything up or if the size<n
return file,size
# Big string: cut it into smaller chunks
file = StringIO(file)
if isinstance(file, FileUpload) and not file:
raise ValueError, 'File not specified'
if hasattr(file, '__class__') and file.__class__ is Pdata:
size=len(file)
return file, size
seek=file.seek
read=file.read
seek(0,2)
size=end=file.tell()
if IBlobStorage.providedBy(self._p_jar.db().storage):
seek(0)
return read(size), size
if size <= 2*n:
seek(0)
if size < n: return read(size), size
return Pdata(read(size)), size
if self._p_jar is None:
# Ugh
seek(0)
return Pdata(read(size)), size
# Now we're going to build a linked list from back
# to front to minimize the number of database updates
# and to allow us to get things out of memory as soon as
# possible.
next = None
while end > 0:
pos = end-n
if pos < n:
pos = 0 # we always want at least n bytes
seek(pos)
# Create the object and assign it a next pointer
# in the same transaction, so that there is only
# a single database update for it.
data = Pdata(read(end-pos))
self._p_jar.add(data)
data.next = next
# Save the object so that we can release its memory.
transaction.savepoint(optimistic=True)
data._p_deactivate()
# The object should be assigned an oid and be a ghost.
assert data._p_oid is not None
assert data._p_state == -1
next = data
end = pos
return next, size
def resave_to_blob(self):
"put the current data in a blob"
data = self.data
if data and not isinstance(data, ZODB.blob.Blob) and IBlobStorage.providedBy(self._p_jar.db().storage):
self.data = ZODB.blob.Blob(str(data))
cache = {}
def blobdirectory():
blobdir = cache.get("blobdir")
if blobdir is None:
blobdir = cache["blobdir"] = mkdtemp()
register(rmtree, blobdir)
return blobdir
# point the test setup to our private `custom_zodb.py`, but only if we
# have to, i.e. for plone 3.x
if not IBlobStorage.providedBy(DemoStorage()):
import Testing
Testing # make pyflakes happy...
import App.config
cfg = App.config.getConfiguration()
cfg.testinghome = dirname(__file__)
# ZopeLite uses DemoStorage directly, so it needs monkey-patching... :(
from Testing.ZopeTestCase import ZopeLite
from ZODB.blob import BlobStorage
def sandbox(base=None):
