def migrate_fileTestesi_to_blob(self):
upgrades.log('Migrating FileTestesi to blob')
walker = CustomQueryWalker(self.context,
FileTestesiMigrator,
full_transaction=True)
walker.go()
upgrades.commit(self.context)
def log_files(self, idx='1'):
ptypes = [
'Image', 'FileTestesi', 'File', 'FileAttachment', 'ImageAttachment'
]
lsize = 0
for ptype in ptypes:
for i in self.context.portal_catalog.search({'portal_type':
ptype}):
obj = i.getObject()
ppath = '/'.join(obj.getPhysicalPath())
field = 'file' in ptype.lower() and 'file' or 'image'
data = obj.getField(field).get(obj).data
ldata = len(data)
lsize += ldata
upgrades.log('%s_AFTERBLOB%s %s;%s;%s;' % (
ptype,
idx,
ppath,
ldata,
hashlib.sha224(data).hexdigest(),
))
upgrades.log('%s_AFTERBLOB%s %s;%s;%s;' % (
'TOTAL SIZE',
idx,
'/t',
lsize,
0,
))
def migrate_simpleattachment_to_blob(self):
"""Move simple attachments to blob"""
upgrades.log('SimpleAttachment -> blob')
upgrades.log(
sa_migrations.migrate_to_blob_storage(self.context)
)
upgrades.commit(self.context)
def to_blob_f(self):
"""To avoir conflicts during the plonemigration loop, run the blob migration now"""
portal = getToolByName(self.context, 'portal_url').getPortalObject()
migrator = FMigrator
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
upgrades.log('Migrating files to blob')
walker.go()
upgrades.commit(self.context)
def migrate_btrees(self):
"""
https://dev.plone.org/ticket/9912
"""
upgrades.log('Migrating to BTrees')
ret = BTreeMigrationView(self.context, self.request)()
upgrades.commit(self.context)
return ret
def to_blob_f(self):
"""To avoir conflicts during the plonemigration loop, run the blob migration now"""
portal = getToolByName(self.context, 'portal_url').getPortalObject()
migrator = FMigrator
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
upgrades.log('Migrating files to blob')
walker.go()
upgrades.commit(self.context)
def migrate_btrees(self):
"""
https://dev.plone.org/ticket/9912
"""
upgrades.log('Migrating to BTrees')
ret = BTreeMigrationView(self.context, self.request)()
upgrades.commit(self.context)
return ret
def postupgrade(self):
"""Mark products as correctly installed by the quickinstaller"""
upgrades.log('postupgrade')
context = self.context
upgrades.quickinstall_addons(self.context,
self.products, [],
upgrades=self.upproducts)
upgrades.commit(context)
def last_migrate_reindex(self):
try:
self.obj.reindexObject(list(self.reindex_indexes))
except Exception, ex:
upgrades.log(
'%s failed reindex during blob migration:\n'
'%s' %
('/'.join(self.obj.getPhysicalPath()), traceback.format_exc()))
def migrate(self):
migration_data = {}
context = self.context
pt = self.context.restrictedTraverse('@@plone_portal_state')
portal_setup = getToolByName(context, 'portal_setup')
context = self.context
portal = pt.portal()
root = pt.navigation_root()
annotations = IAnnotations(portal)
if not PRODUCT in annotations:
annotations[PRODUCT] = PersistentMapping()
migration_data = annotations[PRODUCT]
migration_data.setdefault('migrated', False)
qi = portal.portal_quickinstaller
self.custom_hook_migrationbegin()
self.migrate_btrees()
not_deleted = []
# notneeded: self.delete_manual_broken(not_deleted)
# notneeded: self.delete_broken(not_deleted)
upgrades.cleanup_portal_setup_registries(context)
# notneeded: self.migrate_actions()
# notneeded: self.reimplement()
self.to_blob()
self.custom_hook_blob()
self.to_blob_f()
self.to_blob_i()
self.migrate_object_provides()
self.blob_pack()
self.custom_hook_afterblob()
self.remove_products()
upgrades.cleanup_portal_setup_registries(context)
self.custom_hook_before_plone_upgrade()
self.upgrade_plone()
self.custom_hook_after_plone_upgrade()
self.totinymce()
self.move_custom()
self.update_view_maps()
self.remap_views()
self.remove_ttw()
self.update_products_sets()
self.custom_hook_before_postupgrade()
self.postupgrade()
self.custom_hook_after_postupgrade()
self.cleanup_profile()
# notneeded: self.rebuild_catalog()
upgrades.cleanup_portal_setup_registries(context)
upgrades.remove_persistent_utilities(
context, [re.compile('CacheSetup', re_flags)])
self.custom_hook_aftermigration()
self.final_pack()
self.custom_hook_finished()
migration_data['migrated'] = True
upgrades.log('Migration done')
ret = dict(migration_data.items())
ret[self.path] = dict(migration_data[self.path].items())
return ret
def migrate(self):
migration_data = {}
context = self.context
pt = self.context.restrictedTraverse('@@plone_portal_state')
portal_setup = getToolByName(context, 'portal_setup')
context = self.context
portal = pt.portal()
root = pt.navigation_root()
annotations = IAnnotations(portal)
if not PRODUCT in annotations:
annotations[PRODUCT] = PersistentMapping()
migration_data = annotations[PRODUCT]
migration_data.setdefault('migrated', False)
qi = portal.portal_quickinstaller
self.custom_hook_migrationbegin()
self.migrate_btrees()
not_deleted = []
# notneeded: self.delete_manual_broken(not_deleted)
# notneeded: self.delete_broken(not_deleted)
upgrades.cleanup_portal_setup_registries(context)
# notneeded: self.migrate_actions()
# notneeded: self.reimplement()
self.to_blob()
self.custom_hook_blob()
self.to_blob_f()
self.to_blob_i()
self.migrate_object_provides()
self.blob_pack()
self.custom_hook_afterblob()
self.remove_products()
upgrades.cleanup_portal_setup_registries(context)
self.custom_hook_before_plone_upgrade()
self.upgrade_plone()
self.custom_hook_after_plone_upgrade()
self.totinymce()
self.move_custom()
self.update_view_maps()
self.remap_views()
self.remove_ttw()
self.update_products_sets()
self.custom_hook_before_postupgrade()
self.postupgrade()
self.custom_hook_after_postupgrade()
self.cleanup_profile()
# notneeded: self.rebuild_catalog()
upgrades.cleanup_portal_setup_registries(context)
upgrades.remove_persistent_utilities(
context, [re.compile('CacheSetup', re_flags)])
self.custom_hook_aftermigration()
self.final_pack()
self.custom_hook_finished()
migration_data['migrated'] = True
upgrades.log('Migration done')
ret = dict(migration_data.items())
ret[self.path] = dict(migration_data[self.path].items())
return ret
def last_migrate_reindex(self):
try:
self.obj.reindexObject(list(self.reindex_indexes))
except Exception, ex:
upgrades.log(
'%s failed reindex during blob migration:\n'
'%s' % (
'/'.join(
self.obj.getPhysicalPath()),
traceback.format_exc()))
def upgrade_plone(self):
"""Run the plone_migration tool upgrade loop"""
pm = getToolByName(self.context, 'portal_migration')
if pm.needUpgrading():
upgrades.log(upgrades.upgrade_plone(self.context))
upgrades.commit(self.context)
if pm.needUpgrading():
raise Exception("Plone did not upgrade")
else:
upgrades.commit(self.context)
def wrap(self, *args, **kwargs):
context = self.context
ret = None
path = '/'.join(context.getPhysicalPath())
try:
if not is_migrated(context, step):
ret = callback(self, *args, **kwargs)
else:
upgrades.log('%s already done for %s' % (step, path))
mark_migrated(context, step)
except Exception, e:
raise
def wrap(self, *args, **kwargs):
context = self.context
ret = None
path = '/'.join(context.getPhysicalPath())
try:
if not is_migrated(context, step):
ret = callback(self, *args, **kwargs)
else:
upgrades.log('%s already done for %s' % (step, path))
mark_migrated(context, step)
except Exception, e:
raise
def reimplement(self):
catalog = getToolByName(self.context, 'portal_catalog')
uid_catalog = getToolByName(self.context, 'uid_catalog')
ref_catalog = getToolByName(self.context, 'reference_catalog')
count = 0
for i in catalog.search({}):
o = i.getObject()
if not IIndexableObject.providedBy(o):
count += 1
alsoProvides(o, IIndexableObject)
msg = '%s have been marked as indexable' % count
upgrades.log(msg)
def upgrade_plone(self):
"""Run the plone_migration tool upgrade loop"""
pm = getToolByName(self.context, 'portal_migration')
if pm.needUpgrading():
upgrades.log(
upgrades.upgrade_plone(self.context)
)
upgrades.commit(self.context)
if pm.needUpgrading():
raise Exception("Plone did not upgrade")
else:
upgrades.commit(self.context)
def reimplement(self):
catalog = getToolByName(self.context, 'portal_catalog')
uid_catalog = getToolByName(self.context, 'uid_catalog')
ref_catalog = getToolByName(self.context, 'reference_catalog')
count = 0
for i in catalog.search({}):
o = i.getObject()
if not IIndexableObject.providedBy(o):
count += 1
alsoProvides(o, IIndexableObject)
msg = '%s have been marked as indexable' % count
upgrades.log(msg)
def log_files(self, idx='1'):
ptypes = ['Image', 'FileTestesi', 'File',
'FileAttachment', 'ImageAttachment']
lsize = 0
for ptype in ptypes:
for i in self.context.portal_catalog.search(
{'portal_type': ptype}):
obj = i.getObject()
ppath = '/'.join(obj.getPhysicalPath())
field = 'file' in ptype.lower() and 'file' or 'image'
data = obj.getField(field).get(obj).data
ldata = len(data)
lsize += ldata
upgrades.log(
'%s_AFTERBLOB%s %s;%s;%s;' % (
ptype,
idx, ppath, ldata,
hashlib.sha224(data).hexdigest(),))
upgrades.log(
'%s_AFTERBLOB%s %s;%s;%s;' % (
'TOTAL SIZE', idx, '/t', lsize, 0,))
def migrate_description(self):
context = self.context
catalog = self.context.portal_catalog
portal_transforms = getToolByName(self.context,
'portal_transforms')
upgrades.log('Migrating html description to text')
reindex = []
for itm in catalog.search({}):
obj = itm.getObject()
desc = obj.getField('description').getRaw(obj)
schema = obj.Schema()
if desc:
indexes = []
data = portal_transforms.convert(
'html_to_text', desc
).getData()
if data != desc:
obj.getField('description').set(obj, data)
indexes.append('description')
if 'text' in schema.keys():
text = obj.getField('text').getRaw(obj)
p = '<p> </p>'
if text:
nval = ('%s\n'
'%s\n'
'%s\n'
'%s\n'
'%s') % (
desc, p, p, p, text)
else:
nval = desc
obj.getField('text').set(obj, nval)
indexes.append('text')
# postpone the index to a later step
if indexes:
reindex.append((obj, indexes))
self.migrate_reindexdescription(reindex)
upgrades.log(
'Migrating html description to text: finished')
upgrades.commit(self.context)
def rebuild_catalog(self):
context = self.context
upgrades.upgrade_plone(context)
catalog = getToolByName(self.context, 'portal_catalog')
upgrades.log('Recataloging items')
brains = catalog.search({})
lb = len(brains)
for i, itm in enumerate(brains):
try:
obj = context.unrestrictedTraverse(itm.getPath())
uid = '/'.join(obj.getPhysicalPath())
if not uid in catalog._catalog.uids:
catalog.catalog_object(obj)
catalog.indexObject(obj)
# let the objects be wrapped now by plone.indexer
if IIndexableObject.providedBy(obj):
interface.noLongerProvides(obj, IIndexableObject)
catalog.reindexObject(obj, [
"allowedRolesAndUsers",
"object_provides",
"sortable_title",
"getObjPositionInParent",
"getObjSize",
"is_folderish",
"syndication_enabled",
"is_default_page",
"getIcon",
])
except Exception, e:
upgrades.log('pb cataloging %s; %s' % (itm.getPath(), e))
if i % 10 == 0:
upgrades.log('Recatalog: %s on %s (%s %s)' %
(i, lb, (i / (1.0 * lb) * 100), '%'))
upgrades.commit(context)
def rebuild_catalog(self):
context = self.context
upgrades.upgrade_plone(context)
catalog = getToolByName(self.context, 'portal_catalog')
upgrades.log('Recataloging items')
brains = catalog.search({})
lb = len(brains)
for i, itm in enumerate(brains):
try:
obj = context.unrestrictedTraverse(itm.getPath())
uid = '/'.join(obj.getPhysicalPath())
if not uid in catalog._catalog.uids:
catalog.catalog_object(obj)
catalog.indexObject(obj)
# let the objects be wrapped now by plone.indexer
if IIndexableObject.providedBy(obj):
interface.noLongerProvides(obj, IIndexableObject)
catalog.reindexObject(obj, [
"allowedRolesAndUsers", "object_provides", "sortable_title",
"getObjPositionInParent", "getObjSize", "is_folderish",
"syndication_enabled", "is_default_page", "getIcon",
])
except Exception, e:
upgrades.log('pb cataloging %s; %s' % (itm.getPath(), e))
if i % 10 == 0:
upgrades.log('Recatalog: %s on %s (%s %s)' % (i, lb, (i/(1.0*lb)*100), '%'))
upgrades.commit(context)
def delete_broken(self, not_deleted=None):
if not_deleted is None:
not_deleted = []
catalog = getToolByName(self.context, 'portal_catalog')
uid_catalog = getToolByName(self.context, 'uid_catalog')
ref_catalog = getToolByName(self.context, 'reference_catalog')
not_deleted = []
notstop = True
while notstop:
objects, brokens = self.get_allobjs()
for path in brokens:
if (not self.delete_path(path) and (not path in not_deleted)):
not_deleted.append(path)
objects2, broken2 = self.get_allobjs()
if not len(broken2) < len(brokens):
notstop = False
else:
upgrades.log('Another pass to delete items')
if len(broken2):
catalog.refreshCatalog(clear=1)
uid_catalog.refreshCatalog(clear=1)
ref_catalog.refreshCatalog(clear=1)
upgrades.commit(self.context)
def migrate_description(self):
context = self.context
catalog = self.context.portal_catalog
portal_transforms = getToolByName(self.context, 'portal_transforms')
upgrades.log('Migrating html description to text')
reindex = []
for itm in catalog.search({}):
obj = itm.getObject()
desc = obj.getField('description').getRaw(obj)
schema = obj.Schema()
if desc:
indexes = []
data = portal_transforms.convert('html_to_text',
desc).getData()
if data != desc:
obj.getField('description').set(obj, data)
indexes.append('description')
if 'text' in schema.keys():
text = obj.getField('text').getRaw(obj)
p = '<p> </p>'
if text:
nval = ('%s\n'
'%s\n'
'%s\n'
'%s\n'
'%s') % (desc, p, p, p, text)
else:
nval = desc
obj.getField('text').set(obj, nval)
indexes.append('text')
# postpone the index to a later step
if indexes:
reindex.append((obj, indexes))
self.migrate_reindexdescription(reindex)
upgrades.log('Migrating html description to text: finished')
upgrades.commit(self.context)
def delete_broken(self, not_deleted=None):
if not_deleted is None:
not_deleted = []
catalog = getToolByName(self.context, 'portal_catalog')
uid_catalog = getToolByName(self.context, 'uid_catalog')
ref_catalog = getToolByName(self.context, 'reference_catalog')
not_deleted = []
notstop = True
while notstop:
objects, brokens = self.get_allobjs()
for path in brokens:
if (not self.delete_path(path)
and (not path in not_deleted)):
not_deleted.append(path)
objects2, broken2 = self.get_allobjs()
if not len(broken2) < len(brokens):
notstop = False
else:
upgrades.log('Another pass to delete items')
if len(broken2):
catalog.refreshCatalog(clear=1)
uid_catalog.refreshCatalog(clear=1)
ref_catalog.refreshCatalog(clear=1)
upgrades.commit(self.context)
def delete_path(self, path):
parts = path.split('/')
parent = '/'.join(parts[:-1])
id = parts[-1]
ret = False
try:
parento = self.context.restrictedTraverse(parent)
try:
if parento is None:
raise
parento.manage_delObjects([id])
ret = True
upgrades.log('Deleted %s' % path)
except:
upgrades.log("cannot delete %s" % path)
except:
upgrades.log("Cannot delete %s because parent does not exist anymore" % path)
return ret
def delete_path(self, path):
parts = path.split('/')
parent = '/'.join(parts[:-1])
id = parts[-1]
ret = False
try:
parento = self.context.restrictedTraverse(parent)
try:
if parento is None:
raise
parento.manage_delObjects([id])
ret = True
upgrades.log('Deleted %s' % path)
except:
upgrades.log("cannot delete %s" % path)
except:
upgrades.log(
"Cannot delete %s because parent does not exist anymore" %
path)
return ret
def postupgrade(self):
"""Mark products as correctly installed by the quickinstaller"""
upgrades.log('postupgrade')
context = self.context
upgrades.quickinstall_addons(self.context, self.products, [], upgrades=self.upproducts)
upgrades.commit(context)
def migrate_reindexdescription(self, catalog):
pref = 'Migrating html description to text:'
upgrades.log('%s reindex' % pref)
for obj, indexes in catalog:
obj.reindexObject(indexes)
upgrades.log('%s reindex finished' % pref)
def log(msg, timestamp=True, cr=True):
upgrades.log(msg)
def migrate_simpleattachment_to_blob(self):
"""Move simple attachments to blob"""
upgrades.log('SimpleAttachment -> blob')
upgrades.log(sa_migrations.migrate_to_blob_storage(self.context))
upgrades.commit(self.context)
def migrate_data(self):
ppath = '/'.join(self.obj.getPhysicalPath())
filename = ''
for f in self.fields_map.keys():
_marker = []
oldfield = self.obj.schema[f]
field = self.obj.getPrimaryField()
try:
if ( # already migrated
'blob' in oldfield.storage.get(
oldfield.__name__,
self.obj).__class__.__name__.lower()):
continue
try:
filename = oldfield.get(self.obj).filename
except:
pass
data = oldfield.get(self.obj).data
if (not isinstance(data, basestring)
and hasattr(data, 'data')):
data, odata = '', data
while odata is not None:
data += odata.data
odata = odata.next
if (not data and not isinstance(data, basestring)):
raise Exception('Invalid blob')
except Exception, ex:
upgrades.log('Migration to blob failed for %s:\n'
'%s' % (ppath, traceback.format_exc()))
data = _marker
#upgrades.log(
# '%s %s;%s;%s' %(
# self.dst_portal_type.upper(),
# ppath,
# len(data),
# hashlib.sha224(data).hexdigest(),
# ))
if not data:
upgrades.log('VOID ' + ppath)
resps = self.obj.getResponsables()
self.obj.getField('responsables').set(self.obj, resps)
title = self.obj.Title() or self.obj.title
if not title:
try:
title = self.obj.title_or_id()
except:
pass
if not title:
try:
title = self.obj.getId()
except:
pass
if not title:
try:
title = self.obj.id
except:
pass
self.obj.setTitle(title)
self.obj.title = title
# regarder si ts les idx existent
self.obj.reindexObject([
'Title',
'sortable_title',
])
self.obj.setDescription(self.obj.getDescription()
or self.obj.Description())
if data is not _marker:
fset = field.getMutator(self.obj)
fset(data)
if filename:
field.get(self.obj).setFilename(filename)
def migrate_fileTestesi_to_blob(self):
upgrades.log('Migrating FileTestesi to blob')
walker = CustomQueryWalker(self.context, FileTestesiMigrator, full_transaction=True)
walker.go()
upgrades.commit(self.context)
def pack(self):
upgrades.log('Packing database')
upgrades.commit(self.context)
self.context._p_jar._db.pack()
def migrate_data(self):
ppath = '/'.join( self.obj.getPhysicalPath())
filename = ''
for f in self.fields_map.keys():
_marker = []
oldfield = self.obj.schema[f]
field = self.obj.getPrimaryField()
try:
if ( # already migrated
'blob'
in
oldfield.storage.get(
oldfield.__name__, self.obj
).__class__.__name__.lower()
):
continue
try:
filename = oldfield.get(self.obj).filename
except:
pass
data = oldfield.get(self.obj).data
if (
not isinstance(data, basestring)
and hasattr(data, 'data')
):
data, odata = '', data
while odata is not None:
data += odata.data
odata = odata.next
if (
not data
and not isinstance(data, basestring)
):
raise Exception('Invalid blob')
except Exception, ex:
upgrades.log(
'Migration to blob failed for %s:\n'
'%s' % (ppath , traceback.format_exc()))
data = _marker
#upgrades.log(
# '%s %s;%s;%s' %(
# self.dst_portal_type.upper(),
# ppath,
# len(data),
# hashlib.sha224(data).hexdigest(),
# ))
if not data:
upgrades.log('VOID ' + ppath)
resps = self.obj.getResponsables()
self.obj.getField('responsables').set(self.obj, resps)
title = self.obj.Title() or self.obj.title
if not title:
try:
title = self.obj.title_or_id()
except:
pass
if not title:
try:
title = self.obj.getId()
except:
pass
if not title:
try:
title = self.obj.id
except:
pass
self.obj.setTitle(title)
self.obj.title = title
# regarder si ts les idx existent
self.obj.reindexObject(['Title', 'sortable_title',])
self.obj.setDescription(
self.obj.getDescription()
or self.obj.Description())
if data is not _marker:
fset = field.getMutator(self.obj)
fset(data)
if filename:
field.get(self.obj).setFilename(filename)
def migrate_object_provides(self):
context = self.context
catalog = self.context.portal_catalog
upgrades.log('Migrating object_provides')
catalog.manage_reindexIndex(['object_provides'])
def migrate_reindexdescription(self, catalog):
pref = 'Migrating html description to text:'
upgrades.log('%s reindex' % pref)
for obj, indexes in catalog:
obj.reindexObject(indexes)
upgrades.log('%s reindex finished' % pref)
def log(msg, timestamp=True, cr=True):
upgrades.log(msg)
def migrate_object_provides(self):
context = self.context
catalog = self.context.portal_catalog
upgrades.log('Migrating object_provides')
catalog.manage_reindexIndex(['object_provides'])
def pack(self):
upgrades.log('Packing database')
upgrades.commit(self.context)
self.context._p_jar._db.pack()
