예제 #1
0
 def delete_broken(self, not_deleted=None):
     if not_deleted is None:
         not_deleted = []
     catalog = getToolByName(self.context, 'portal_catalog')
     uid_catalog = getToolByName(self.context, 'uid_catalog')
     ref_catalog = getToolByName(self.context, 'reference_catalog')
     not_deleted = []
     notstop = True
     while notstop:
         objects, brokens = self.get_allobjs()
         for path in brokens:
             if (
                 not self.delete_path(path)
                 and (not path in not_deleted)
             ):
                 not_deleted.append(path)
         objects2, broken2 = self.get_allobjs()
         if not len(broken2) < len(brokens):
             notstop = False
         else:
             upgrades.log('Another pass to delete items')
     if len(broken2):
         catalog.refreshCatalog(clear=1)
         uid_catalog.refreshCatalog(clear=1)
         ref_catalog.refreshCatalog(clear=1)
     upgrades.commit(self.context)
예제 #2
0
 def postupgrade(self):
     """Mark products as correctly installed by the quickinstaller"""
     upgrades.log('postupgrade')
     context = self.context
     upgrades.quickinstall_addons(
         self.context, self.products, [], upgrades=self.upproducts)
     upgrades.commit(context)
예제 #3
0
 def migrate_btrees(self):
     """
     https://dev.plone.org/ticket/9912
     """
     upgrades.log('Migrating to BTrees')
     ret = BTreeMigrationView(self.context, self.request)()
     upgrades.commit(self.context)
     return ret
예제 #4
0
 def to_blob_f(self):
     """To avoir conflicts during the plonemigration loop, run the blob migration now"""
     portal = getToolByName(self.context, 'portal_url').getPortalObject()
     migrator = FMigrator
     walker = CustomQueryWalker(portal, migrator, full_transaction=True)
     upgrades.log('Migrating files to blob')
     walker.go()
     upgrades.commit(self.context)
예제 #5
0
 def reimplement(self):
     catalog = getToolByName(self.context, 'portal_catalog')
     count = 0
     for i in catalog.search({}):
         o = i.getObject()
         if not IIndexableObject.providedBy(o):
             count += 1
             alsoProvides(o, IIndexableObject)
     msg = '%s have been marked as indexable' % count
     upgrades.log(msg)
예제 #6
0
 def last_migrate_reindex(self):
     try:
         self.obj.reindexObject(list(self.reindex_indexes))
     except Exception:
         upgrades.log(
             '%s failed reindex during blob migration:\n'
             '%s' % (
                 '/'.join(
                     self.obj.getPhysicalPath()),
                 traceback.format_exc()))
예제 #7
0
 def wrap(self, *args, **kwargs):
     context = self.context
     path = '/'.join(context.getPhysicalPath())
     try:
         if not is_migrated(context, step):
             callback(self, *args, **kwargs)
         else:
             upgrades.log('%s already done for %s' % (step, path))
         mark_migrated(context, step)
     except Exception:
         raise
예제 #8
0
 def upgrade_plone(self):
     """Run the plone_migration tool upgrade loop"""
     pm = getToolByName(self.context, 'portal_migration')
     if pm.needUpgrading():
         upgrades.log(
             upgrades.upgrade_plone(self.context)
         )
         upgrades.commit(self.context)
     if pm.needUpgrading():
         raise Exception("Plone did not upgrade")
     else:
         upgrades.commit(self.context)
예제 #9
0
def control_wv(view):
    upgrades.log("Wvhtml killer/watcher thread start")
    onow = None
    onow = now = datetime.datetime.now()
    while view.controlrun:
        now = datetime.datetime.now()
        if now - onow > datetime.timedelta(seconds=30):
            upgrades.log("Always running wv control thread")
            onow = now
            pl = subprocess.Popen(
                ['ps', '-aeo', 'etime,pid,command'],
                stdout=subprocess.PIPE).communicate()[0]
        for i in [a.strip() for a in pl.splitlines() if 'wv' in a.lower()]:
            etime = i.split()[0]
            pid = i.split()[1]
            if ":" in etime:
                minutes = etime.split(':')[0]
                try:
                    if int(minutes) > 1:
                        subprocess.Popen(
                            ['kilL', '-9', '%s' % pid],
                            stdout=subprocess.PIPE).communicate()[0]
                        upgrades.log("Wvhtml control: kill %s" % pid)
                except:
                    pass
        time.sleep(10)
    upgrades.log("Wvhtml killer/watcher thread end")
예제 #10
0
    def migrate_description(self):
        catalog = self.context.portal_catalog
        portal_transforms = getToolByName(self.context,
                                          'portal_transforms')
        upgrades.log('Migrating html description to text')
        reindex = []
        for itm in catalog.search({}):
            obj = itm.getObject()
            desc = obj.getField('description').getRaw(obj)
            schema = obj.Schema()
            if desc:
                indexes = []
                data = portal_transforms.convert(
                    'html_to_text', desc
                ).getData()
                if data != desc:
                    obj.getField('description').set(obj, data)
                    indexes.append('description')
                if 'text' in schema.keys():
                    text = obj.getField('text').getRaw(obj)
                    p = '<p>&nbsp;</p>'
                    if text:
                        nval = ('%s\n'
                                '%s\n'
                                '%s\n'
                                '%s\n'
                                '%s') % (
                            desc, p, p, p, text)
                    else:
                        nval = desc
                    obj.getField('text').set(obj, nval)
                    indexes.append('text')
                # postpone the index to a later step
                if indexes:

                    reindex.append((obj, indexes))
        self.migrate_reindexdescription(reindex)
        upgrades.log(
            'Migrating html description to text: finished')
        upgrades.commit(self.context)
예제 #11
0
 def rebuild_catalog(self):
     context = self.context
     upgrades.upgrade_plone(context)
     catalog = getToolByName(self.context, 'portal_catalog')
     upgrades.log('Recataloging items')
     brains = catalog.search({})
     lb = len(brains)
     for i, itm in enumerate(brains):
         try:
             obj = context.unrestrictedTraverse(itm.getPath())
             uid = '/'.join(obj.getPhysicalPath())
             if not uid in catalog._catalog.uids:
                 catalog.catalog_object(obj)
                 catalog.indexObject(obj)
             # let the objects be wrapped now by plone.indexer
             if IIndexableObject.providedBy(obj):
                 interface.noLongerProvides(obj, IIndexableObject)
                 catalog.reindexObject(obj, [
                     "allowedRolesAndUsers", "object_provides", "sortable_title",
                     "getObjPositionInParent", "getObjSize", "is_folderish",
                     "syndication_enabled", "is_default_page", "getIcon"]
                 )
         except Exception, e:
             upgrades.log('pb cataloging %s; %s' % (itm.getPath(), e))
         if i % 10 == 0:
             upgrades.log('Recatalog: %s on %s (%s %s)' % (
                 i, lb, (i / (1.0 * lb) * 100), '%'))
             upgrades.commit(context)
예제 #12
0
 def log_files(self, idx='1'):
     ptypes = ['Image', 'FileIbwtPlonesite', 'File',
               'FileAttachment', 'ImageAttachment']
     lsize = 0
     for ptype in ptypes:
         for i in self.context.portal_catalog.search(
             {'portal_type': ptype}
         ):
             obj = i.getObject()
             ppath = '/'.join(obj.getPhysicalPath())
             field = 'file' in ptype.lower() and 'file' or 'image'
             data = obj.getField(field).get(obj).data
             ldata = len(data)
             lsize += ldata
             upgrades.log(
                 '%s_AFTERBLOB%s %s;%s;%s;' % (
                     ptype,
                     idx, ppath, ldata,
                     hashlib.sha224(data).hexdigest(),))
     upgrades.log(
         '%s_AFTERBLOB%s %s;%s;%s;' % (
             'TOTAL SIZE', idx, '/t', lsize, 0,))
예제 #13
0
 def delete_path(self, path):
     parts = path.split('/')
     parent = '/'.join(parts[:-1])
     id = parts[-1]
     ret = False
     try:
         parento = self.context.restrictedTraverse(parent)
         try:
             if parento is None:
                 raise
             parento.manage_delObjects([id])
             ret = True
             upgrades.log('Deleted %s' % path)
         except:
             upgrades.log("cannot delete %s" % path)
     except:
         upgrades.log("Cannot delete %s because parent does not exist anymore" % path)
     return ret
예제 #14
0
 def migrate_object_provides(self):
     catalog = self.context.portal_catalog
     upgrades.log('Migrating object_provides')
     catalog.manage_reindexIndex(['object_provides'])
예제 #15
0
 def migrate_reindexdescription(self, catalog):
     pref = 'Migrating html description to text:'
     upgrades.log('%s reindex' % pref)
     for obj, indexes in catalog:
         obj.reindexObject(indexes)
     upgrades.log('%s reindex finished' % pref)
예제 #16
0
 def migrate_fileIbwtPlonesite_to_blob(self):
     upgrades.log('Migrating FileIbwtPlonesite to blob')
     walker = CustomQueryWalker(self.context, FileIbwtPlonesiteMigrator, full_transaction=True)
     walker.go()
     upgrades.commit(self.context)
예제 #17
0
 def migrate_data(self):
     ppath = '/'.join(self.obj.getPhysicalPath())
     filename = ''
     for f in self.fields_map.keys():
         _marker = []
         oldfield = self.obj.schema[f]
         field = self.obj.getPrimaryField()
         try:
             # already migrated
             if (
                 'blob' in
                 oldfield.storage.get(
                     oldfield.__name__, self.obj
                 ).__class__.__name__.lower()
             ):
                 continue
             try:
                 filename = oldfield.get(self.obj).filename
             except:
                 pass
             data = oldfield.get(self.obj).data
             if (
                 not isinstance(data, basestring)
                 and hasattr(data, 'data')
             ):
                 data, odata = '', data
                 while odata is not None:
                     data += odata.data
                     odata = odata.next
             if (
                 not data
                 and not isinstance(data, basestring)
             ):
                 raise Exception('Invalid blob')
         except Exception:
             upgrades.log(
                 'Migration to blob failed for %s:\n'
                 '%s' % (ppath, traceback.format_exc()))
             data = _marker
         #upgrades.log(
         #        '%s %s;%s;%s' %(
         #            self.dst_portal_type.upper(),
         #            ppath,
         #            len(data),
         #            hashlib.sha224(data).hexdigest(),
         #        ))
         if not data:
             upgrades.log('VOID ' + ppath)
         resps = self.obj.getResponsables()
         self.obj.getField('responsables').set(self.obj, resps)
         title = self.obj.Title() or self.obj.title
         if not title:
             try:
                 title = self.obj.title_or_id()
             except:
                 pass
         if not title:
             try:
                 title = self.obj.getId()
             except:
                 pass
         if not title:
             try:
                 title = self.obj.id
             except:
                 pass
         self.obj.setTitle(title)
         self.obj.title = title
         # regarder si ts les idx existent
         self.obj.reindexObject(['Title', 'sortable_title'])
         self.obj.setDescription(
             self.obj.getDescription()
             or self.obj.Description())
         if data is not _marker:
             fset = field.getMutator(self.obj)
             fset(data)
             if filename:
                 field.get(self.obj).setFilename(filename)
예제 #18
0
 def log(msg, timestamp=True, cr=True):
     upgrades.log(msg)
예제 #19
0
 def migrate(self):
     migration_data = {}
     context = self.context
     # launch wvhtml controller thread to avoid dangling wv processes
     self.controlrun = True
     control_wv_t = threading.Thread(target=control_wv, args=(self,))
     control_wv_t.start()
     pt = self.context.restrictedTraverse('@@plone_portal_state')
     #portal_setup = getToolByName(context, 'portal_setup')
     context = self.context
     portal = pt.portal()
     #root = pt.navigation_root()
     annotations = IAnnotations(portal)
     if not PRODUCT in annotations:
         annotations[PRODUCT] = PersistentMapping()
     migration_data = annotations[PRODUCT]
     migration_data.setdefault('migrated', False)
     #qi = portal.portal_quickinstaller
     self.custom_hook_migrationbegin()
     self.migrate_btrees()
     #not_deleted = []
     # notneeded: self.delete_manual_broken(not_deleted)
     # notneeded: self.delete_broken(not_deleted)
     upgrades.cleanup_portal_setup_registries(context)
     # notneeded: self.migrate_actions()
     # notneeded: self.reimplement()
     self.to_blob()
     self.custom_hook_blob()
     self.to_blob_f()
     self.to_blob_i()
     self.migrate_object_provides()
     self.blob_pack()
     self.custom_hook_afterblob()
     self.remove_products()
     upgrades.cleanup_portal_setup_registries(context)
     self.custom_hook_before_plone_upgrade()
     self.upgrade_plone()
     self.custom_hook_after_plone_upgrade()
     self.totinymce()
     self.move_custom()
     self.update_view_maps()
     self.remap_views()
     self.remove_ttw()
     self.update_products_sets()
     self.custom_hook_before_postupgrade()
     self.postupgrade()
     self.custom_hook_after_postupgrade()
     self.cleanup_profile()
     # notneeded: self.rebuild_catalog()
     upgrades.cleanup_portal_setup_registries(context)
     upgrades.remove_persistent_utilities(
         context, [re.compile('CacheSetup', re_flags)])
     self.custom_hook_aftermigration()
     self.final_pack()
     self.custom_hook_finished()
     migration_data['migrated'] = True
     upgrades.log('Migration done')
     self.controlrun = False
     control_wv_t.join(1)
     ret = dict(migration_data.items())
     ret[self.path] = dict(migration_data[self.path].items())
     return ret
예제 #20
0
 def pack(self):
     upgrades.log('Packing database')
     upgrades.commit(self.context)
     self.context._p_jar._db.pack()