def getFields(self, obj):
if not self.fields:
# get the blob fields to migrate from the first object
for field in ISchema(obj).fields():
if IBlobField.providedBy(field):
self.fields.append(field.getName())
return self.fields
def handleDeferred(self, obj, objpath, content):
"""
very large files get deferred to get sent out
so we need an extra handler for it
"""
if 'fieldvalues' in content:
for fieldname, value in content['fieldvalues'].items():
if value['value'] == json.Deferred:
resp = requests.post(self.sourcesite + \
'/@@migrator-exportfield',
data={'path': objpath, 'field': fieldname})
largefile = False
if int(resp.headers['content-length']) / 1024 / 1024 > 50:
largefile = True
transaction.commit() # commit before and after
migr = FieldMigrator(self.site, obj, fieldname)
field = obj.getField(fieldname)
content = resp.content
filename = resp.headers.get('filename', '')
mimetype = resp.headers.get('content-type', '')
if IBlobField.providedBy(field):
# not a blob field here...
content = StringIO(content)
content.filename = filename
migr.set({
'value': content,
'extras': {
'filename': filename,
'mimetype': mimetype
}
})
if largefile:
transaction.commit()
def handleDeferred(self, obj, objpath, content):
"""
very large files get deferred to get sent out
so we need an extra handler for it
"""
if 'fieldvalues' in content:
for fieldname, value in content['fieldvalues'].items():
if value['value'] == json.Deferred:
resp = requests.post(self.sourcesite + \
'/@@migrator-exportfield',
data={'path': objpath, 'field': fieldname})
largefile = False
if int(resp.headers['content-length']) / 1024 / 1024 > 50:
largefile = True
transaction.commit() # commit before and after
migr = FieldMigrator(self.site, obj, fieldname)
field = obj.getField(fieldname)
content = resp.content
filename = resp.headers.get('filename', '')
mimetype = resp.headers.get('content-type', '')
if IBlobField.providedBy(field):
# not a blob field here...
content = StringIO(content)
content.filename = filename
migr.set({'value': content, 'extras': {
'filename': filename,
'mimetype': mimetype
}})
if largefile:
transaction.commit()
def iterFields(ob):
# noinspection PyUnresolvedReferences
ob = Acquisition.aq_base(ob)
primary = ob.getPrimaryField()
if primary:
clone = cloneField(primary, primary=True)
yield clone.__name__, clone
for name in ob.schema.getSchemataNames():
for field in ob.schema.getSchemataFields(name):
if primary and primary.__name__ == field.__name__:
continue
# Mark 'primary fields', which get marshaled into payload
if (bool(getattr(field, 'primary', None)) is True
or IBlobField.providedBy(field)
or (IFileField.providedBy(field)
and not ITextField.providedBy(field))
or getattr(field, 'widget', None) == RichWidget
or isinstance(field, ZPTField)):
clone = cloneField(field, primary=True)
else:
clone = cloneField(field, primary=False)
yield clone.__name__, clone
def iterFields(ob):
# noinspection PyUnresolvedReferences
ob = Acquisition.aq_base(ob)
primary = ob.getPrimaryField()
if primary:
clone = cloneField(primary, primary=True)
yield clone.__name__, clone
for name in ob.schema.getSchemataNames():
for field in ob.schema.getSchemataFields(name):
if primary and primary.__name__ == field.__name__:
continue
# Mark 'primary fields', which get marshaled into payload
if (bool(getattr(field, 'primary', None)) is True
or IBlobField.providedBy(field)
or (IFileField.providedBy(field)
and not ITextField.providedBy(field))
or getattr(field, 'widget', None) == RichWidget
or isinstance(field, ZPTField)):
clone = cloneField(field, primary=True)
else:
clone = cloneField(field, primary=False)
yield clone.__name__, clone
def extractFile(self, obj, field):
""" Return tuple of (filename, content_type, data)
"""
field = obj.getField(field)
if HAS_BLOB and IBlobField.providedBy(field):
wrapper = field.getRaw(obj)
value = str(wrapper)
fname = wrapper.getFilename()
if fname is None:
fname = ''
ct = wrapper.getContentType()
if ct is None:
ct = ''
else:
# temporarily:
# dirty call, I know, just lazy to get method arguments
# TextField overrided getBaseUnit method but didn't follow API
try:
base_unit = field.getBaseUnit(obj, full=True)
except TypeError:
base_unit = field.getBaseUnit(obj)
fname = base_unit.getFilename()
ct = base_unit.getContentType()
value = base_unit.getRaw()
return fname, ct, value
def getFields(self, obj):
if not self.fields:
# get the blob fields to migrate from the first object
for field in ISchema(obj).fields():
if IBlobField.providedBy(field):
self.fields.append(field.getName())
return self.fields
def extractFile(self, obj, field):
""" Return tuple of (filename, content_type, data)
"""
field = obj.getField(field)
if HAS_BLOB and IBlobField.providedBy(field):
wrapper = field.getRaw(obj)
value = str(wrapper)
fname = wrapper.getFilename()
if fname is None:
fname = ''
ct = wrapper.getContentType()
if ct is None:
ct = ''
else:
# temporarily:
# dirty call, I know, just lazy to get method arguments
# TextField overrided getBaseUnit method but didn't follow API
try:
base_unit = field.getBaseUnit(obj, full=True)
except TypeError:
base_unit = field.getBaseUnit(obj)
fname = base_unit.getFilename()
ct = base_unit.getContentType()
value = base_unit.getRaw()
return fname, ct, value
def _deploy_content(self, obj, is_page=True):
"""
Deploy object as page.
"""
try:
new_req, orig_req = fakeRequest(obj)
except AttributeError:
# not a valid obj to override request with
new_req = None
content = self._render_obj(obj)
if content is None:
return
filename = obj.absolute_url_path().lstrip("/")
# deploy additional views for content type
if PLONE_APP_BLOB_INSTALLED and isinstance(obj, ATBlob):
self._deploy_views([os.path.join(filename, "view")], is_page=True)
if is_page:
filename = filename.rstrip("/")
if self.add_index or IFolder.providedBy(obj):
filename = os.path.join(filename, "index.html")
elif not filename.endswith(".htm") and not filename.endswith(".html"):
filename = filename + ".html"
elif (
isinstance(obj, ATImage)
or hasattr(obj, "getBlobWrapper")
and "image" in obj.getBlobWrapper().getContentType()
):
# create path to dump ATImage in original size
if filename.rsplit(".", 1)[-1] in ("png", "jpg", "gif", "jpeg"):
filename = os.path.join(filename, "image.%s" % (filename.rsplit(".", 1)[-1]))
else:
filename = os.path.join(filename, "image.jpg")
filename, content = self._apply_image_transforms(filename, content)
elif hasattr(obj, "getBlobWrapper") and "image" not in obj.getBlobWrapper().getContentType():
# create path like for ATImage
if len(filename.rsplit(".", 1)) > 1:
filename = os.path.join(filename, "file.%s" % (filename.rsplit(".", 1)[-1]))
else:
filename = os.path.join(filename, "file")
self._write(filename, content)
# deploy all sizes of images uploaded for the object
if not getattr(obj, "schema", None):
return
for field in obj.Schema().fields():
if (PLONE_APP_BLOB_INSTALLED and IBlobImageField.providedBy(field)) or field.type == "image":
self._deploy_blob_image_field(obj, field)
elif PLONE_APP_BLOB_INSTALLED and IBlobField.providedBy(field):
self._deploy_blob_file_field(obj, field)
elif field.type == "file" and obj.meta_type not in self.file_types:
self._deploy_file_field(obj, field)
else:
continue
if new_req is not None:
restoreRequest(orig_req, new_req)
def fieldFilter(field):
return (
(
IBlobField.providedBy(field) or
IFileField.providedBy(field) or
IImageField.providedBy(field)
) and
not ITextField.providedBy(field)
)
def afterRetrieveModifier(self, obj, repo_clone, preserve=()):
"""If we find any BlobProxies, replace them with the values
from the current working copy."""
blob_fields = (f for f in obj.Schema().fields() if IBlobField.providedBy(f))
for f in blob_fields:
blob = f.getUnwrapped(obj, raw=True).getBlob()
clone_ref = f.getUnwrapped(repo_clone, raw=True)
if isinstance(clone_ref.blob, BlobProxy):
clone_ref.setBlob(blob)
return [], [], {}
def __call__(self, recursive=True):
# look for a content with name file field into content
field = self.context.getPrimaryField()
for field in [self.context.getPrimaryField()] \
+ self.context.Schema().values():
if IBlobField.providedBy(field) or IFileField.providedBy(field):
return self.context.getField('file').get(self.context)
#@TODO: should work with a basic IFileField
else:
return None
def __new__index_html(self, REQUEST=None, RESPONSE=None):
"""Make it directly viewable when entering the objects URL
"""
field = self.getPrimaryField()
if not IBlobField.providedBy(field) and IMultilanguageField.providedBy(field):
if REQUEST is None and hasattr(self, 'REQUEST'):
REQUEST = self.REQUEST
if REQUEST is not None and not 'lang' in REQUEST.keys():
url = REQUEST['ACTUAL_URL']
url += urlparse(url).query and '&' or '?'
url += 'lang='+field._getCurrentLanguage(self)
return REQUEST.response.redirect(url)
return self.__old__index_html(REQUEST, RESPONSE)
def __new__index_html(self, REQUEST=None, RESPONSE=None):
"""Make it directly viewable when entering the objects URL
"""
field = self.getPrimaryField()
if not IBlobField.providedBy(field) and IMultilanguageField.providedBy(
field):
if REQUEST is None and hasattr(self, 'REQUEST'):
REQUEST = self.REQUEST
if REQUEST is not None and not 'lang' in REQUEST.keys():
url = REQUEST['ACTUAL_URL']
url += urlparse(url).query and '&' or '?'
url += 'lang=' + field._getCurrentLanguage(self)
return REQUEST.response.redirect(url)
return self.__old__index_html(REQUEST, RESPONSE)
def _deploy_content(self, obj, is_page=True):
"""
Deploy object as page.
"""
content = self._render_obj(obj)
if content is None:
return
filename = obj.absolute_url_path().lstrip('/')
# deploy additional views for content type
if PLONE_APP_BLOB_INSTALLED and isinstance(obj, ATBlob):
self._deploy_views([os.path.join(filename, 'view'), ],
is_page=True)
if is_page:
filename = os.path.join(filename, 'index.html')
elif isinstance(obj, ATImage) or hasattr(obj, 'getBlobWrapper') and 'image' in obj.getBlobWrapper().getContentType():
# create path to dump ATImage in original size
if filename.rsplit('.', 1)[-1] in ('png', 'jpg', 'gif', 'jpeg'):
filename = os.path.join(filename, 'image.%s' % filename.rsplit('.', 1)[-1])
else:
filename = os.path.join(filename, 'image.jpg')
filename, content = self._apply_image_transforms(filename, content)
elif (hasattr(obj, 'getBlobWrapper') and 'image' not in
obj.getBlobWrapper().getContentType()):
# create path like for ATImage
if len(filename.rsplit('.', 1)) > 1:
filename = os.path.join(filename, 'file.%s' % filename.rsplit('.', 1)[-1])
else:
filename = os.path.join(filename, 'file')
self._write(filename, content)
# deploy all sizes of images uploaded for the object
if not getattr(obj, 'schema', None):
return
for field in obj.Schema().fields():
if PLONE_APP_BLOB_INSTALLED and IBlobImageField.providedBy(field):
self._deploy_blob_image_field(obj, field)
elif PLONE_APP_BLOB_INSTALLED and IBlobField.providedBy(field):
self._deploy_blob_file_field(obj, field)
elif field.type == 'image':
self._deploy_image_field(obj, field)
elif field.type == 'file' and obj.meta_type not in self.file_types:
self._deploy_file_field(obj, field)
else:
continue
def getOnCloneModifiers(self, obj):
"""Removes blob objects and stores a marker
"""
blob_refs = dict((id(f.getUnwrapped(obj, raw=True).getBlob()), True)
for f in obj.Schema().fields()
if IBlobField.providedBy(f))
def persistent_id(obj):
if id(aq_base(obj)) in blob_refs:
return True
return None
def persistent_load(ignored):
return BlobProxy()
return persistent_id, persistent_load, [], []
def getReferencedAttributes(self, obj):
blob_fields = (f for f in obj.Schema().fields()
if IBlobField.providedBy(f))
file_data = {}
# try to get last revision, only store a new blob if the
# contents differ from the prior one, otherwise store a
# reference to the prior one
# XXX: According to CopyModifyMergeRepository, retrieve and getHistory
# should not be used as a part of more complex transactions
# due to some resource managers not supporting savepoints.
repo = getToolByName(obj, 'portal_repository')
try:
prior_rev = repo.retrieve(obj)
except ArchivistRetrieveError:
prior_rev = None
for f in blob_fields:
# XXX: should only do this if data has changed since last
# version somehow Resave the blob line by line
blob_file = f.get(obj, raw=True).getBlob().open('r')
save_new = True
if prior_rev is not None:
prior_obj = prior_rev.object
prior_blob = f.get(prior_obj, raw=True).getBlob()
prior_file = prior_blob.open('r')
# Check for file size differences
if (os.fstat(prior_file.fileno()).st_size ==
os.fstat(blob_file.fileno()).st_size):
# Files are the same size, compare line by line
for line, prior_line in izip(blob_file, prior_file):
if line != prior_line:
break
else:
# The files are the same, save a reference
# to the prior versions blob on this version
file_data[f.getName()] = prior_blob
save_new = False
if save_new:
new_blob = file_data[f.getName()] = Blob()
new_blob_file = new_blob.open('w')
try:
blob_file.seek(0)
new_blob_file.writelines(blob_file)
finally:
blob_file.close()
new_blob_file.close()
return file_data
def getOnCloneModifiers(self, obj):
"""Removes references to blobs.
"""
blob_refs = dict(
(id(f.getUnwrapped(obj, raw=True).getBlob()), True)
for f in obj.Schema().fields()
if IBlobField.providedBy(f)
)
def persistent_id(obj):
if id(aq_base(obj)) in blob_refs:
return True
return None
def persistent_load(obj):
return None
return persistent_id, persistent_load, [], []
def fieldFilter(field):
return ((IBlobField.providedBy(field)
or IFileField.providedBy(field)
or IImageField.providedBy(field))
and not ITextField.providedBy(field))
def unserializeFields(self, object, jsonkey):
"""
Unserializes the fielddata and optimizes it of the modifiers of
the fields.
Gets and sets from / to **self.data**
"""
# we just have to go throught if we have a dict, with possible
# schema fields
if not isinstance(self.data[jsonkey], dict):
return self.data
if IPloneSiteRoot.providedBy(object):
return self.data
if not hasattr(aq_base(object), 'Schema'):
# might be dexterity
return self.data
fields = object.Schema().fields()
for field in fields:
name = field.getName()
if name not in self.data[jsonkey].keys():
continue
# DateTimeField doesnt need to be converted t DateTime
# FileFields are base64 encoded
if HAS_BLOBS and IBlobField.providedBy(field) or \
isinstance(field, (ImageField, FileField)):
value = self.data[jsonkey][name]
if isinstance(value, dict) and not value['data']:
self.data[jsonkey][name] = None
elif isinstance(value, dict):
# decode it
data = StringIO(base64.decodestring(value['data']))
data.seek(0)
setattr(data, 'filename', value['filename'])
self.data[jsonkey][name] = data
# ReferenceField: remove bad UIDs
if isinstance(field, ReferenceField):
# check if field ist multiValued
if field.multiValued:
cleaned = []
for uid in self.data[jsonkey][name]:
obj = self.context.reference_catalog.lookupObject(uid)
if obj:
cleaned.append(uid)
else:
if uid:
self.logger.warn(
('The reference field <%s> of object(%s)'
' has a broken reference to'
' the object %s') % (name,
object.UID(),
uid))
self.data[jsonkey][name] = cleaned
else:
cleaned = None
obj = self.context.reference_catalog.lookupObject(
self.data[jsonkey][name])
if obj:
cleaned = self.data[jsonkey][name]
else:
if self.data[jsonkey][name]:
self.logger.warn(
('The reference field <%s> of object(%s) has a'
' broken reference to the object %s') % (
name,
object.UID(),
self.data[jsonkey][name]))
self.data[jsonkey][name] = cleaned
# ImageField: treat empty files special
if isinstance(field, ImageField):
if not self.data[jsonkey][name]:
self.data[jsonkey][name] = 'DELETE_IMAGE'
# FileField (direct): treat empty files special
if field.__class__ == FileField:
if not self.data[jsonkey][name]:
self.data[jsonkey][name] = 'DELETE_FILE'
return self.data
def _deploy_content(self, obj, is_page=True):
"""
Deploy object as page.
"""
try:
new_req, orig_req = fakeRequest(obj)
except AttributeError:
# not a valid obj to override request with
new_req = None
content = self._render_obj(obj)
if content is None:
return
filename = obj.absolute_url_path().lstrip('/')
# deploy additional views for content type
if PLONE_APP_BLOB_INSTALLED and isinstance(obj, ATBlob):
self._deploy_views([os.path.join(filename, 'view'), ],
is_page=True)
if is_page:
filename = filename.rstrip('/')
if self.add_index or IFolder.providedBy(obj):
filename = os.path.join(filename, 'index.html')
elif not filename.endswith('.htm') and not filename.endswith('.html'):
filename = filename + '.html'
elif isinstance(obj, ATImage) or \
hasattr(obj, 'getBlobWrapper') and \
'image' in obj.getBlobWrapper().getContentType():
# create path to dump ATImage in original size
if filename.rsplit('.', 1)[-1] in ('png', 'jpg', 'gif', 'jpeg'):
filename = os.path.join(filename, 'image.%s' % (
filename.rsplit('.', 1)[-1]))
else:
filename = os.path.join(filename, 'image.jpg')
filename, content = self._apply_image_transforms(filename, content)
elif (hasattr(obj, 'getBlobWrapper') and 'image' not in
obj.getBlobWrapper().getContentType()):
# create path like for ATImage
if len(filename.rsplit('.', 1)) > 1:
filename = os.path.join(filename, 'file.%s' % (
filename.rsplit('.', 1)[-1]))
else:
filename = os.path.join(filename, 'file')
self._write(filename, content)
# deploy all sizes of images uploaded for the object
if not getattr(obj, 'schema', None):
return
# For Dexterity objects
if IDexterityContent.providedBy(obj):
from plone.dexterity.interfaces import IDexterityFTI
from zope.component import getUtility
from zope.schema import getFieldsInOrder
from plone.behavior.interfaces import IBehaviorAssignable
fti = getUtility(IDexterityFTI, name=obj.portal_type)
schema = fti.lookupSchema()
fields = getFieldsInOrder(schema)
for _, field in fields:
if INamedImageField.providedBy(field):
self._deploy_blob_dexterity_image_field(obj, field)
elif INamedFileField.providedBy(field):
self._deploy_blob_dexterity_file_field(obj, field)
behavior_assignable = IBehaviorAssignable(obj)
if behavior_assignable:
behaviors = behavior_assignable.enumerateBehaviors()
for behavior in behaviors:
for k, v in getFieldsInOrder(behavior.interface):
pass
else:
for field in obj.Schema().fields():
if (PLONE_APP_BLOB_INSTALLED and IBlobImageField.providedBy(field)) or \
field.type == 'image':
self._deploy_blob_image_field(obj, field)
elif PLONE_APP_BLOB_INSTALLED and IBlobField.providedBy(field):
self._deploy_blob_file_field(obj, field)
elif field.type == 'file' and obj.portal_type not in self.file_types:
self._deploy_file_field(obj, field)
else:
continue
if new_req is not None:
restoreRequest(orig_req, new_req)
def get_list_of_blobs(self):
""" Get list of all blobs with their OID
"""
logger.info('Start report')
# query = {'portal_type':{
# 'query':[
# 'Article',
# 'Blob' ,
# 'DataFile',
# 'EEAFigureFile',
# 'FactSheetDocument',
# 'File',
# 'FlashFile',
# 'Highlight',
# 'Image',
# 'PressRelease',
# 'Promotion',
# 'Report'
# 'Speech',
# ],
# 'operator':'or'
# }}
query = {
'Language': 'all',
}
tree = {}
cat = getToolByName(self, 'portal_catalog', None)
brains = cat(**query)
logger.info('%d objects will to be processed', len(brains))
for brain in brains:
obj = brain.getObject()
schema = getattr(obj.aq_inner.aq_self, 'schema', None)
if not schema:
continue
fields = [f for f in schema.fields() if IBlobField.providedBy(f)]
if not len(fields) and obj.getField('file'):
fields = [obj.getField('file')]
for f in fields:
bw = f.getRaw(obj)
blob = bw.getBlob()
oid = blob._p_oid
serial = blob._p_serial
conn = Globals.DB.open()
file_path = conn._storage.fshelper.getBlobFilename(oid, serial)
conn.close()
try:
full_path = blob.committed()
except Exception:
full_path = 'missing blob'
tree[oid_repr(blob._p_oid)] = (f.getName(),
full_path,
file_path,
brain.portal_type,
brain.getURL())
logger.info('Done report!')
return pprint.pformat(tree)
def is_blob(field):
return IBlobField.providedBy(field)
def is_blob(field):
return IBlobField.providedBy(field)