def migrate_fileTestesi_to_blob(self):
upgrades.log('Migrating FileTestesi to blob')
walker = CustomQueryWalker(self.context,
FileTestesiMigrator,
full_transaction=True)
walker.go()
upgrades.commit(self.context)
def migrate(context):
# switch linkintegrity temp off
ptool = queryUtility(IPropertiesTool)
site_props = getattr(ptool, 'site_properties', None)
linkintegrity = site_props.getProperty(
'enable_link_integrity_checks',
False
)
site_props.manage_changeProperties(enable_link_integrity_checks=False)
# do migration
portal = getToolByName(context, 'portal_url').getPortalObject()
walker = CustomQueryWalker(
portal, PloneArticleMigrator,
query=dict(object_provides=IPloneArticle.__identifier__),
callBefore=callBefore)
savepoint(optimistic=True)
walker.go()
# switch linkintegrity back
site_props.manage_changeProperties(
enable_link_integrity_checks=linkintegrity
)
return walker.getOutput()
def migrate_fileVirtualcollections_to_blob(self):
upgrades.log('Migrating FileVirtualcollections to blob')
walker = CustomQueryWalker(
self.context,
FileVirtualcollectionsMigrator,
full_transaction=True)
walker.go()
upgrades.commit(self.context)
def to_blob_f(self):
"""To avoir conflicts during the plonemigration loop, run the blob migration now"""
portal = getToolByName(self.context, 'portal_url').getPortalObject()
migrator = FMigrator
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
upgrades.log('Migrating files to blob')
walker.go()
upgrades.commit(self.context)
def to_blob_f(self):
"""To avoir conflicts during the plonemigration loop, run the blob migration now"""
portal = getToolByName(self.context, 'portal_url').getPortalObject()
migrator = FMigrator
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
upgrades.log('Migrating files to blob')
walker.go()
upgrades.commit(self.context)
def migrate_fileTestCgwb_to_blob(self):
upgrades.log('Migrating FileTestCgwb to blob')
walker = CustomQueryWalker(
self.context,
FileTestCgwbMigrator,
full_transaction=True)
walker.go()
upgrades.commit(self.context)
def migrate_filePhenologyPlone_to_blob(self):
upgrades.log('Migrating FilePhenologyPlone to blob')
walker = CustomQueryWalker(
self.context,
FilePhenologyPloneMigrator,
full_transaction=True)
walker.go()
upgrades.commit(self.context)
def upgrade_step_1(context):
portal = getToolByName(context, 'portal_url').getPortalObject()
walker = CustomQueryWalker(
portal, PAEATMigrator,
query=dict(object_provides=IATEvent.__identifier__),
callBefore=callBefore)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrateSmartLinkToLink(context):
from Products.contentmigration.walker import CustomQueryWalker
migrator = makeMigrator(context, 'Link', 'ATLink')
walker = CustomQueryWalker(context,
migrator,
callBefore=isSmartLink,
use_savepoint=True)
walker.go()
return walker.getOutput().splitlines()
def execute(self, query, actions, callBefore=None, **kwargs):
TestMigrator.fieldActions = actions
walker = CustomQueryWalker(self.portal, TestMigrator,
query = query,
callBefore = callBefore,
**kwargs)
# Need this to avoid copy errors....
transaction.savepoint(optimistic=True)
walker.go(**kwargs)
def execute(self, query, actions, callBefore=None, **kwargs):
TestMigrator.fieldActions = actions
walker = CustomQueryWalker(self.portal, TestMigrator,
query = query,
callBefore = callBefore,
**kwargs)
# Need this to avoid copy errors....
transaction.savepoint(optimistic=True)
walker.go(**kwargs)
def upgrade_step_1(context):
portal = getToolByName(context, 'portal_url').getPortalObject()
walker = CustomQueryWalker(
portal,
PAEATMigrator,
query=dict(object_provides=IATEvent.__identifier__),
callBefore=callBefore)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrate(context, portal_type=None, meta_type=None, walker=None):
""" migrate instances using the given walker """
if walker is None:
migrator = makeMigrator(context, portal_type, meta_type)
walker = CustomQueryWalker(context, migrator, full_transaction=True)
else:
walker = walker(context)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrate(context, portal_type=None, meta_type=None, walker=None):
""" migrate instances using the given walker """
if walker is None:
migrator = makeMigrator(context, portal_type, meta_type)
walker = CustomQueryWalker(context, migrator, full_transaction=True)
else:
walker = walker(context)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrate(self):
out = StringIO()
print >> out, "Starting ratings migration"
portal_url = getToolByName(self, 'portal_url')
portal = portal_url.getPortalObject()
# Migrate release count variable
walker = CustomQueryWalker(portal, RatingsMigrator,
query = {'portal_type': 'PSCProject'})
transaction.savepoint(optimistic=True)
print >> out, "Switching from contentratings to twothumbs.."
walker.go(out=out)
print >> out, walker.getOutput()
print >> out, "Migration finished"
def __call__(self):
res = 'State before:\n' + self.stats() + '\n'
portal = self.context
for migrator in [ATBlobFileToAWSFileMigrator, ATBlobImageToAWSImageMigrator]:
max_size = self.request.get('max_size')
def callBefore(obj, **kwargs):
return obj.get_size() < int(max_size) if max_size else True
walker = CustomQueryWalker(portal, migrator, callBefore=callBefore, use_savepoint=True)
if self.request.get('limit'):
walker.limit = int(self.request.get('limit'))
transaction.savepoint(optimistic=True)
walker.go()
res += walker.getOutput()
res += 'State after:\n' + self.stats() + '\n'
portal.plone_log(res)
return res
def migrate(self):
out = StringIO()
print >> out, "Starting ratings migration"
portal_url = getToolByName(self, 'portal_url')
portal = portal_url.getPortalObject()
# Migrate release count variable
walker = CustomQueryWalker(portal,
RatingsMigrator,
query={'portal_type': 'PSCProject'})
transaction.savepoint(optimistic=True)
print >> out, "Switching from contentratings to twothumbs.."
walker.go(out=out)
print >> out, walker.getOutput()
print >> out, "Migration finished"
def migrateFlowplayerToRedTurtleVideo(context):
from Products.contentmigration.walker import CustomQueryWalker
migrator = makeMigrator(context, 'ATFile')
walker = CustomQueryWalker(
context,
migrator,
use_savepoint=True,
query={'object_provides': IVideo.__identifier__})
walker.go()
output = walker.getOutput()
migrator = makeMigrator(context, 'ATBlob')
walker = CustomQueryWalker(
context,
migrator,
use_savepoint=True,
query={'object_provides': IVideo.__identifier__})
walker.go()
output += walker.getOutput()
return output.splitlines()
def migrateContents(context, src_type, dst_type, src_metatype, dst_metatype, query={}):
from Products.contentmigration.walker import CustomQueryWalker
#BBB: i can't find a better way to know if a given portal_type is folderish or not
is_folderish = False
temp_obj = context.restrictedTraverse('portal_factory/%s/tmp_id' % src_type)
if temp_obj:
plone_view = temp_obj.restrictedTraverse('@@plone')
if plone_view.isStructuralFolder():
is_folderish = True
portal_types = context.portal_types
src_infos = portal_types.getTypeInfo(src_type)
dst_infos = portal_types.getTypeInfo(dst_type)
if is_folderish:
migrator = makeFolderMigrator(context,
src_type,
dst_type,
src_metatype,
dst_metatype)
else:
migrator = makeContentMigrator(context,
src_type,
dst_type,
src_metatype,
dst_metatype)
if migrator:
if not src_metatype:
src_metatype = src_infos.content_meta_type
if not dst_metatype:
dst_metatype = dst_infos.content_meta_type
migrator.src_meta_type = src_metatype
migrator.dst_meta_type = dst_metatype
walker = CustomQueryWalker(context, migrator,
src_portal_type=src_type,
dst_portal_type=dst_type,
src_meta_type=src_metatype,
dst_meta_type=dst_metatype,
query=query,
use_savepoint=True)
walker.go()
walk_infos = {'error': walker.errors,
'msg': walker.getOutput().splitlines(),
'counter': walker.counter}
return walk_infos
def migrateFlowplayerToRedTurtleVideo(context):
from Products.contentmigration.walker import CustomQueryWalker
migrator = makeMigrator(context, 'ATFile')
walker = CustomQueryWalker(context, migrator, use_savepoint=True,
query={'object_provides': IVideo.__identifier__})
walker.go()
output = walker.getOutput()
migrator = makeMigrator(context, 'ATBlob')
walker = CustomQueryWalker(context, migrator, use_savepoint=True,
query={'object_provides': IVideo.__identifier__})
walker.go()
output += walker.getOutput()
return output.splitlines()
def migrateContents(context, src_type, dst_type, query={}):
from Products.contentmigration.walker import CustomQueryWalker
#BBB: i can't find a better way to know if a given portal_type is folderish or not
is_folderish = False
temp_obj = context.restrictedTraverse('portal_factory/%s/tmp_id' %
src_type)
if temp_obj:
plone_view = temp_obj.restrictedTraverse('@@plone')
if plone_view.isStructuralFolder():
is_folderish = True
portal_types = context.portal_types
src_infos = portal_types.getTypeInfo(src_type)
dst_infos = portal_types.getTypeInfo(dst_type)
if is_folderish:
migrator = makeFolderMigrator(
context,
src_type,
dst_type,
)
else:
migrator = makeContentMigrator(
context,
src_type,
dst_type,
)
if migrator:
migrator.src_meta_type = src_infos.content_meta_type
migrator.dst_meta_type = dst_infos.content_meta_type
walker = CustomQueryWalker(context,
migrator,
src_portal_type=src_type,
dst_portal_type=dst_type,
use_savepoint=True,
query=query)
walker.go()
walk_infos = {
'error': walker.errors,
'msg': walker.getOutput().splitlines(),
'counter': walker.counter
}
return walk_infos
def migrate(self):
"""Run the migration"""
out = StringIO()
print >> out, "Starting migration"
portal_url = getToolByName(self, 'portal_url')
portal = portal_url.getPortalObject()
migrators = (MyMigrator,)
for migrator in migrators:
walker = CustomQueryWalker(portal, migrator,
query = {'path' : find(self)})
# walker = migrator.walkerClass(portal, migrator)
walker.go(out=out)
print >> out, walker.getOutput()
print >> out, "Migration finished"
return out.getvalue()
def migrate(self, target):
out = StringIO()
catalog = getToolByName(self,'portal_catalog')
portal_url = getToolByName(self, 'portal_url')
portal = portal_url.getPortalObject()
id = self.id
path = '/'.join(self.getParentNode().getPhysicalPath())
results = catalog(meta_type="FedoraHierarchie", getId=id, path={'query':path, 'depth':1})
if target == 'Issue':
migrator = IssueMigrator
elif target == 'Volume':
migrator = VolumeMigrator
walker = CustomQueryWalker(portal, migrator, query = {'path' : {'query':path, 'depth':1}, 'getId':id})
walker.go(out=out)
print >> out, walker.getOutput()
return out.getvalue()
def upgrade_step_1(context):
# switch linkintegrity temp off
ptool = queryUtility(IPropertiesTool)
site_props = getattr(ptool, 'site_properties', None)
linkintegrity = site_props.getProperty('enable_link_integrity_checks',
False)
site_props.manage_changeProperties(enable_link_integrity_checks=False)
# do migration
portal = getToolByName(context, 'portal_url').getPortalObject()
walker = CustomQueryWalker(
portal,
PAEATMigrator,
query=dict(object_provides=IATEvent.__identifier__),
callBefore=callBefore)
savepoint(optimistic=True)
walker.go()
# switch linkintegrity back
site_props.manage_changeProperties(
enable_link_integrity_checks=linkintegrity)
return walker.getOutput()
def from_1_1_to_1_2(self, out):
"""
Migrate from 1.1 to 1.2: We now use a field `instructors' instead
of `instructor', and it contains a list instead of a string.
"""
if not haveContentMigrations:
print >> out, "WARNING: Install contentmigrations to be able to migrate from 1.1 to 1.2"
return
class InstructorsMigrator(InlineFieldActionMigrator):
src_portal_type = src_meta_type = (ECL_META,)
fieldActions = ({ 'fieldName': 'instructor',
'newFieldName': 'instructors',
'transform': instructor_to_instructors,
},
)
# Migrate instructor field
portal = getToolByName(self, 'portal_url').getPortalObject()
walker = CustomQueryWalker(portal, InstructorsMigrator, query={})
transaction.savepoint(optimistic=True)
print >> out, "Migrating from instructor to instructors"
walker.go()
def migrate_topics(portal):
"""Migrate ATContentTypes Topics to plone.app.contenttypes Collections.
This could also be used as upgrade step.
"""
# Parse the registry to get allowed operations and pass it to the
# migrator.
reg = getUtility(IRegistry)
reader = IQuerystringRegistryReader(reg)
registry = reader.parseRegistry()
# select migrator based on the base-class of collections
fti = portal.portal_types['Collection']
if fti.content_meta_type == "Dexterity Item":
migrator = TopicMigrator
elif fti.content_meta_type == "Dexterity Container":
migrator = FolderishTopicMigrator
walker = CustomQueryWalker(portal, migrator)(registry=registry)
return walker
def migrateSimpleAttachment(portal, migrator):
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrateCustomAT(fields_mapping, src_type, dst_type, dry_run=False):
"""
Try to get types infos from archetype_tool, then set a migrator an pass it
given values. There is a dry_run mode that allows to check the success of
a migration without committing.
"""
portal = getSite()
# if the type still exists get the src_meta_type from the portal_type
portal_types = getToolByName(portal, 'portal_types')
fti = portal_types.get(src_type, None)
# Check if the fti was removed or replaced by a DX-implementation
if fti is None or IDexterityFTI.providedBy(fti):
# Get the needed info from an instance of the type
catalog = portal.portal_catalog
brains = catalog(portal_type=src_type, sort_limit=1)
if not brains:
# no item? assume stuff
is_folderish = False
src_meta_type = src_type
else:
src_obj = brains[0].getObject()
if IDexterityContent.providedBy(src_obj):
logger.error(
'%s should not be dexterity object!',
src_obj.absolute_url())
is_folderish = getattr(src_obj, 'isPrincipiaFolderish', False)
src_meta_type = src_obj.meta_type
else:
# Get info from at-fti
src_meta_type = fti.content_meta_type
archetype_tool = getToolByName(portal, 'archetype_tool', None)
for info in archetype_tool.listRegisteredTypes():
# lookup registered type in archetype_tool with meta_type
# because several portal_types can use same meta_type
if info.get('meta_type') == src_meta_type:
klass = info.get('klass', None)
is_folderish = klass.isPrincipiaFolderish
migrator = makeCustomATMigrator(context=portal,
src_type=src_type,
dst_type=dst_type,
fields_mapping=fields_mapping,
is_folderish=is_folderish,
dry_run=dry_run)
if migrator:
migrator.src_meta_type = src_meta_type
migrator.dst_meta_type = ''
walker_settings = {'portal': portal,
'migrator': migrator,
'src_portal_type': src_type,
'dst_portal_type': dst_type,
'src_meta_type': src_meta_type,
'dst_meta_type': '',
'use_savepoint': True}
if dry_run:
walker_settings['limit'] = 1
walker = CustomQueryWalker(**walker_settings)
walker.go()
walker_infos = {'errors': walker.errors,
'msg': walker.getOutput().splitlines(),
'counter': walker.counter}
for error in walker.errors:
logger.error(error.get('message'))
if dry_run:
transaction.abort()
return walker_infos
def migrateCustomAT(fields_mapping, src_type, dst_type, dry_run=False):
"""
Try to get types infos from archetype_tool, then set a migrator and pass it
given values. There is a dry_run mode that allows to check the success of
a migration without committing.
"""
portal = getSite()
# if the type still exists get the src_meta_type from the portal_type
portal_types = getToolByName(portal, 'portal_types')
fti = portal_types.get(src_type, None)
# Check if the fti was removed or replaced by a DX-implementation
if fti is None or IDexterityFTI.providedBy(fti):
# Get the needed info from an instance of the type
catalog = portal.portal_catalog
brains = catalog(portal_type=src_type, sort_limit=1)
if not brains:
# no item? assume stuff
is_folderish = False
src_meta_type = src_type
else:
src_obj = brains[0].getObject()
if IDexterityContent.providedBy(src_obj):
logger.error('%s should not be dexterity object!',
src_obj.absolute_url())
is_folderish = getattr(src_obj, 'isPrincipiaFolderish', False)
src_meta_type = src_obj.meta_type
else:
# Get info from at-fti
src_meta_type = fti.content_meta_type
archetype_tool = getToolByName(portal, 'archetype_tool', None)
for info in archetype_tool.listRegisteredTypes():
# lookup registered type in archetype_tool with meta_type
# because several portal_types can use same meta_type
if info.get('meta_type') == src_meta_type:
klass = info.get('klass', None)
is_folderish = klass.isPrincipiaFolderish
migrator = makeCustomATMigrator(context=portal,
src_type=src_type,
dst_type=dst_type,
fields_mapping=fields_mapping,
is_folderish=is_folderish,
dry_run=dry_run)
if migrator:
migrator.src_meta_type = src_meta_type
migrator.dst_meta_type = ''
walker_settings = {
'portal': portal,
'migrator': migrator,
'src_portal_type': src_type,
'dst_portal_type': dst_type,
'src_meta_type': src_meta_type,
'dst_meta_type': '',
'use_savepoint': True
}
if dry_run:
walker_settings['limit'] = 1
walker = CustomQueryWalker(**walker_settings)
walker.go()
walker_infos = {
'errors': walker.errors,
'msg': walker.getOutput().splitlines(),
'counter': walker.counter
}
for error in walker.errors:
logger.error(error.get('message'))
if dry_run:
transaction.abort()
return walker_infos
def v1beta7_v1beta8(self, out):
"""Migrate from beta 7 to beta 8
"""
if not haveContentMigrations:
print >> out, "WARNING: Install contentmigrations to be able to migrate from v1 beta 7 to beta 8"
return
class ReleaseStateMigrator(BaseInlineMigrator):
src_portal_type = src_meta_type = "PSCRelease"
stateMap = {
"final": "final",
"rc": "release-candidate",
"beta": "beta",
"alpha": "alpha",
"in progress": "pre-release",
"in-progress": "pre-release",
}
def migrate_releaseState(self):
maturity = getattr(aq_base(self.obj), "maturity", None)
wftool = getToolByName(self.obj, "portal_workflow")
wfdef = getattr(aq_base(wftool), "psc_release_workflow", None)
if maturity is not None and wfdef is not None:
maturity = str(maturity)
state = self.stateMap.get(maturity.lower(), "pre-release")
wfstate = {
"action": None,
"actor": None,
"comments": "Updated by migration; maturity was " + maturity,
"review_state": state,
"time": DateTime(),
}
wftool.setStatusOf("psc_release_workflow", self.obj, wfstate)
wfdef.updateRoleMappingsFor(self.obj)
self.obj.reindexObject()
class ReleaseCountMigrator(BaseInlineMigrator):
src_portal_type = src_meta_type = "PSCProject"
def migrate_releaseCount(self):
releaseCount = getattr(aq_base(self.obj), "releaseCount", None)
catalog = getToolByName(self.obj, "portal_catalog")
releases = catalog.searchResults(
portal_type="PSCRelease",
review_state=("alpha", "beta", "release-candidate", "final"),
path="/".join(self.obj.getPhysicalPath()),
)
if releaseCount is None:
self.obj.manage_addProperty("releaseCount", len(releases), "int")
else:
self.obj.manage_changeProperties(releaseCount=len(releases))
self.obj.reindexObject()
portal = getToolByName(self, "portal_url").getPortalObject()
# Migrate release state
walker = CustomQueryWalker(portal, ReleaseStateMigrator, query={})
transaction.savepoint(optimistic=True)
print >> out, "Migrating from field-based maturity to workflow-based maturity"
walker.go()
# Migrate release count variable
walker = CustomQueryWalker(portal, ReleaseCountMigrator, query={})
transaction.savepoint(optimistic=True)
print >> out, "Adding release count property"
walker.go()
def getMigrationWalker(context, migrator):
""" set up migration walker using the given item migrator """
portal = getToolByName(context, 'portal_url').getPortalObject()
return CustomQueryWalker(portal, migrator, use_savepoint=True)
def v1beta7_v1beta8(self, out):
"""Migrate from beta 7 to beta 8
"""
if not haveContentMigrations:
print >> out, "WARNING: Install contentmigrations to be able to migrate from v1 beta 7 to beta 8"
return
class ReleaseStateMigrator(BaseInlineMigrator):
src_portal_type = src_meta_type = 'PSCRelease'
stateMap = {'final' : 'final',
'rc' : 'release-candidate',
'beta' : 'beta',
'alpha' : 'alpha',
'in progress' : 'pre-release',
'in-progress' : 'pre-release'}
def migrate_releaseState(self):
maturity = getattr(aq_base(self.obj), 'maturity', None)
wftool = getToolByName(self.obj, 'portal_workflow')
wfdef = getattr(aq_base(wftool), 'psc_release_workflow', None)
if maturity is not None and wfdef is not None:
maturity = str(maturity)
state = self.stateMap.get(maturity.lower(), 'pre-release')
wfstate = {'action' : None,
'actor' : None,
'comments' : 'Updated by migration; maturity was ' + maturity,
'review_state' : state,
'time' : DateTime()}
wftool.setStatusOf('psc_release_workflow', self.obj, wfstate)
wfdef.updateRoleMappingsFor(self.obj)
self.obj.reindexObject()
class ReleaseCountMigrator(BaseInlineMigrator):
src_portal_type = src_meta_type = 'PSCProject'
def migrate_releaseCount(self):
releaseCount = getattr(aq_base(self.obj), 'releaseCount', None)
catalog = getToolByName(self.obj, 'portal_catalog')
releases = catalog.searchResults(portal_type = 'PSCRelease',
review_state = ('alpha', 'beta', 'release-candidate', 'final',),
path = '/'.join(self.obj.getPhysicalPath()))
if releaseCount is None:
self.obj.manage_addProperty('releaseCount', len(releases), 'int')
else:
self.obj.manage_changeProperties(releaseCount = len(releases))
self.obj.reindexObject()
portal = getToolByName(self, 'portal_url').getPortalObject()
# Migrate release state
walker = CustomQueryWalker(portal, ReleaseStateMigrator, query = {})
transaction.savepoint(optimistic=True)
print >> out, "Migrating from field-based maturity to workflow-based maturity"
walker.go()
# Migrate release count variable
walker = CustomQueryWalker(portal, ReleaseCountMigrator, query = {})
transaction.savepoint(optimistic=True)
print >> out, "Adding release count property"
walker.go()
def migrateSimpleAttachment(portal, migrator):
walker = CustomQueryWalker(portal, migrator, full_transaction=True)
savepoint(optimistic=True)
walker.go()
return walker.getOutput()
def migrateSmartLinkToLink(context):
from Products.contentmigration.walker import CustomQueryWalker
migrator = makeMigrator(context, 'Link', 'ATLink')
walker = CustomQueryWalker(context, migrator, callBefore=isSmartLink, use_savepoint=True)
walker.go()
return walker.getOutput().splitlines()