def migrateObjects(self, objects):
paths = []
for obj in objects:
objpath = self.getRelativePath(obj)
if objpath not in self.imported and not \
(self.onlyNew and objpath in self.site._import_results):
paths.append(objpath)
if paths: # only if there are ones to migrate
migr = MultiContentObjectMigrator(self.site, self.site,
paths, self.attributes)
response = requests.post(self.source, data={
'migrator': migr.title,
'args': json.dumps({
'attributes': self.attributes,
'paths': paths})
})
objectsData = json.loads(response.content)
for path, content in objectsData.items():
path = str(path)
object = safeTraverse(self.site, path)
self._migrateObject(object, path, content=content)
for obj in objects:
# then need to check if any are folders...
if IBaseFolder.providedBy(obj):
migr = FolderContentsMigrator(self.site, obj)
folderdata = requests.post(self.source, data={
'migrator': FolderContentsMigrator.title,
'path': '/'.join(obj.getPhysicalPath()
)[len(self.sitepath) + 1:]
})
self(migr, json.loads(folderdata.content))
def getMigratorFromRequest(request):
from wildcard.migrator import getMigrator
migrator = request.get('migrator')
if not migrator:
raise Exception("Must specify a migrator")
migrator = getMigrator(migrator)
site = getSite()
args = request.get('args')
if args:
args = json.loads(args)
for key, val in args.items():
del args[key]
args[str(key)] = val
else:
args = {}
if migrator._type in ['object', 'folder', '_']:
path = request.get('path', '')
context = safeTraverse(site, str(path), None)
if context is None:
redirect_storage = getUtility(IRedirectionStorage)
site_path = '/'.join(site.getPhysicalPath())
newpath = redirect_storage.get(
site_path + '/' + path.lstrip('/'), None)
if newpath:
context = safeTraverse(site, newpath, None)
if not context:
context = path
migrator = migrator(site, context, **args)
else:
migrator = migrator(site, None, **args)
return migrator
def migrateObjects(self, objects):
paths = []
for obj in objects:
objpath = self.getRelativePath(obj)
if objpath not in self.imported and not \
(self.onlyNew and objpath in self.site._import_results):
paths.append(objpath)
if paths: # only if there are ones to migrate
migr = MultiContentObjectMigrator(self.site, self.site, paths,
self.attributes)
response = requests.post(self.source,
data={
'migrator':
migr.title,
'args':
json.dumps({
'attributes': self.attributes,
'paths': paths
})
})
objectsData = json.loads(response.content)
for path, content in objectsData.items():
path = str(path)
object = safeTraverse(self.site, path)
self._migrateObject(object, path, content=content)
for obj in objects:
# then need to check if any are folders...
if IBaseFolder.providedBy(obj):
migr = FolderContentsMigrator(self.site, obj)
folderdata = requests.post(
self.source,
data={
'migrator':
FolderContentsMigrator.title,
'path':
'/'.join(obj.getPhysicalPath())[len(self.sitepath) +
1:]
})
self(migr, json.loads(folderdata.content))
def migrateObject(self, obj, content=None):
objpath = self.getRelativePath(obj)
if objpath not in self.imported and not \
(self.onlyNew and objpath in self.site._import_results):
self._migrateObject(obj, objpath, content=content)
if IBaseFolder.providedBy(obj):
migr = FolderContentsMigrator(self.site, obj)
folderdata = requests.post(self.source, data={
'migrator': FolderContentsMigrator.title,
'path': '/'.join(obj.getPhysicalPath()
)[len(self.sitepath) + 1:]
})
self(migr, json.loads(folderdata.content))
def _touchPath(self, path):
# have data for request but no object created yet?
# need to assemble obj first then
obj = safeTraverse(self.site, path, None)
if obj:
return obj
resp = requests.post(self.source, data={
'migrator': ContentTouchMigrator.title,
'path': path
})
content = json.loads(resp.content)
migr = ContentTouchMigrator(self.site, path)
return migr.set(content)
def _migrateObject(self, obj, objpath, content=None):
if content is None:
response = requests.post(self.source,
data={
'migrator':
ContentObjectMigrator.title,
'path':
objpath,
'args':
json.dumps(
{'attributes': self.attributes})
})
content = json.loads(response.content)
totouch = []
for uid, path in content['uids']:
if uid not in self.convertedUids:
path = str(path).lstrip('/')
uidObj = None
if path in self.stubs or path in self.imported:
# might be imported but we don't know the uid conversion...
uidObj = safeTraverse(self.site, path, None)
if uidObj:
self.convertedUids[uid] = uidObj.UID()
if uidObj is None:
# create stub object if they aren't there
# this is so we can convert uids
totouch.append((path, uid))
#self.touchPath(path, uid)
if totouch:
self.touchPaths(totouch)
self.convertUids(content)
logger.info('apply data migrations on %s' % (objpath))
migr = ContentObjectMigrator(self.site, obj)
error = True
while error:
error = False
try:
migr.set(content)
self.handleDeferred(obj, objpath, content)
except MissingObjectException, ex:
logger.info('oops, could not find %s - touching' % ex.path)
path = ex.path
try:
self._touchPath(path)
except ValueError:
# error in response. must not be valid object
if path not in self.site._touch_errors:
self.site._touch_errors.append(path)
error = True
def _touchPath(self, path):
# have data for request but no object created yet?
# need to assemble obj first then
obj = safeTraverse(self.site, path, None)
if obj:
return obj
resp = requests.post(self.source,
data={
'migrator': ContentTouchMigrator.title,
'path': path
})
content = json.loads(resp.content)
migr = ContentTouchMigrator(self.site, path)
return migr.set(content)
def migrateObject(self, obj, content=None):
objpath = self.getRelativePath(obj)
if objpath not in self.imported and not \
(self.onlyNew and objpath in self.site._import_results):
self._migrateObject(obj, objpath, content=content)
if IBaseFolder.providedBy(obj):
migr = FolderContentsMigrator(self.site, obj)
folderdata = requests.post(
self.source,
data={
'migrator':
FolderContentsMigrator.title,
'path':
'/'.join(obj.getPhysicalPath())[len(self.sitepath) + 1:]
})
self(migr, json.loads(folderdata.content))
def _migrateObject(self, obj, objpath, content=None):
if content is None:
response = requests.post(self.source, data={
'migrator': ContentObjectMigrator.title,
'path': objpath,
'args': json.dumps({
'attributes': self.attributes})
})
content = json.loads(response.content)
totouch = []
for uid, path in content['uids']:
if uid not in self.convertedUids:
path = str(path).lstrip('/')
uidObj = None
if path in self.stubs or path in self.imported:
# might be imported but we don't know the uid conversion...
uidObj = safeTraverse(self.site, path, None)
if uidObj:
self.convertedUids[uid] = uidObj.UID()
if uidObj is None:
# create stub object if they aren't there
# this is so we can convert uids
totouch.append((path, uid))
#self.touchPath(path, uid)
if totouch:
self.touchPaths(totouch)
self.convertUids(content)
logger.info('apply data migrations on %s' % (objpath))
migr = ContentObjectMigrator(self.site, obj)
error = True
while error:
error = False
try:
migr.set(content)
self.handleDeferred(obj, objpath, content)
except MissingObjectException, ex:
logger.info(
'oops, could not find %s - touching' % ex.path)
path = ex.path
try:
self._touchPath(path)
except ValueError:
# error in response. must not be valid object
if path not in self.site._touch_errors:
self.site._touch_errors.append(path)
error = True
def touchPaths(self, uids):
totouch = []
for path, uid in uids:
obj = safeTraverse(self.site, path, None)
if not obj:
totouch.append((path, uid))
else:
self.convertedUids[uid] = obj.UID()
resp = requests.post(self.source, data={
'migrator': MultiContentTouchMigrator.title,
'args': json.dumps({'totouch': totouch})
})
content = json.loads(resp.content)
migr = MultiContentTouchMigrator(self.site)
for touched, olduid in migr.set(content):
path = '/'.join(touched.getPhysicalPath())[len(self.sitepath) + 1:]
self.stubs.append(path.lstrip('/'))
realuid = touched.UID()
self.convertedUids[olduid] = realuid
def touchPaths(self, uids):
totouch = []
for path, uid in uids:
obj = safeTraverse(self.site, path, None)
if not obj:
totouch.append((path, uid))
else:
self.convertedUids[uid] = obj.UID()
resp = requests.post(self.source,
data={
'migrator': MultiContentTouchMigrator.title,
'args': json.dumps({'totouch': totouch})
})
content = json.loads(resp.content)
migr = MultiContentTouchMigrator(self.site)
for touched, olduid in migr.set(content):
path = '/'.join(touched.getPhysicalPath())[len(self.sitepath) + 1:]
self.stubs.append(path.lstrip('/'))
realuid = touched.UID()
self.convertedUids[olduid] = realuid
def __call__(self):
if not hasattr(self.context, '_import_results'):
self.context._import_results = PersistentList()
if not hasattr(self.context, '_touch_errors'):
self.context._touch_errors = PersistentList()
if self.request.get('REQUEST_METHOD') == 'POST':
sourcesite = self.request.get('source', '').rstrip('/')
if not sourcesite:
raise Exception("Must specify a source")
migratorname = self.request.get('migrator')
migrator = getMigratorFromRequest(self.request)
source = sourcesite + '/@@migrator-exporter'
attributes = self.request.get('attributes', '').splitlines()
result = requests.post(
source, data={'migrator': self.request.get('migrator')})
data = json.loads(result.content)
if self.request.get('onlyNew', False):
onlyNew = True
else:
onlyNew = False
if self.request.get('index', False):
index = True
else:
index = False
if migratorname == SiteContentsMigrator.title:
threshold = int(self.request.get('threshold', '150'))
batch = int(self.request.get('batch', '1'))
contentmigrator = ContentMigrator(self.request,
source,
sourcesite,
migrator.site,
threshold=threshold,
attributes=attributes,
onlyNew=onlyNew,
index=index,
batch=batch)
contentmigrator(migrator, data)
else:
migrator.set(data)
return 'done'
return self.template()
def __call__(self):
if not hasattr(self.context, '_import_results'):
self.context._import_results = PersistentList()
if not hasattr(self.context, '_touch_errors'):
self.context._touch_errors = PersistentList()
if self.request.get('REQUEST_METHOD') == 'POST':
sourcesite = self.request.get('source', '').rstrip('/')
if not sourcesite:
raise Exception("Must specify a source")
migratorname = self.request.get('migrator')
migrator = getMigratorFromRequest(self.request)
source = sourcesite + '/@@migrator-exporter'
attributes = self.request.get('attributes', '').splitlines()
result = requests.post(source, data={
'migrator': self.request.get('migrator')})
data = json.loads(result.content)
if self.request.get('onlyNew', False):
onlyNew = True
else:
onlyNew = False
if self.request.get('index', False):
index = True
else:
index = False
if migratorname == SiteContentsMigrator.title:
threshold = int(self.request.get('threshold', '150'))
batch = int(self.request.get('batch', '1'))
contentmigrator = ContentMigrator(self.request, source,
sourcesite, migrator.site,
threshold=threshold,
attributes=attributes,
onlyNew=onlyNew,
index=index,
batch=batch)
contentmigrator(migrator, data)
else:
migrator.set(data)
return 'done'
return self.template()