def recursive_create_path(path):
if path in ('/', ''):
folder = site
else:
path = path.lstrip('/')
folder = traverse(path)
if folder is not None and not IFolder.providedBy(folder):
api.content.delete(folder)
folder = None
if folder is None:
# Need to create folders up to where we want content
# we'll walk it up create folders as needed
folder = site
for part in path.split('/'):
try:
ob = folder[part]
if not IFolder.providedBy(ob):
api.content.delete(ob)
raise KeyError()
else:
folder = ob
except (KeyError, AttributeError):
fpath = os.path.join(args.export_directory,
relpath(folder), part, '__folder__')
if os.path.exists(fpath):
fi = open(fpath)
data = mjson.loads(fi.read())
fi.close()
importtype = get_import_type(
data, fpath[len(args.export_directory):], None)
creation_data = importtype.get_data()
creation_data['container'] = folder
creation_data['id'] = part
try:
folder = api.content.create(**creation_data)
except api.exc.InvalidParameterError:
logger.error(
'Error creating content {}'.format(fpath),
exc_info=True)
return
importtype.post_creation(folder)
if data['state']:
try:
api.content.transition(folder,
to_state=data['state'])
except:
# maybe workflows do not match up
pass
folder.reindexObject()
else:
folder = api.content.create(type='Folder',
id=part,
title=part.capitalize(),
container=folder)
bdata = ILayoutAware(folder)
bdata.contentLayout = '++contentlayout++castle/folder-query.html'
return folder
def import_object(self, filepath, container=None):
fi = open(filepath)
file_read = fi.read()
fi.close()
try:
data = mjson.loads(file_read)
except Exception:
print("Skipping {}; Unable to read JSON data".format(filepath))
return
if filepath.endswith('__folder__'):
filepath = '/'.join(filepath.split('/')[:-1])
skipped = False
if data['portal_type'] in skip_types:
print('Skipping omitted type {type}'.format(
type=data['portal_type']))
skipped = True
if only_types and data['portal_type'] not in only_types:
print("Skipping {type} at {path}, not in only_types.".format(
type=data['portal_type'], path=filepath))
skipped = True
if import_paths:
do_import = False
for import_path in import_paths:
if filepath.startswith('{}/{}'.format(args.export_directory,
import_path)):
do_import = True
if import_path.startswith(
filepath[len(args.export_directory):].lstrip('/') +
'/'):
# Don't skip folders on the way to import_paths
do_import = True
if not do_import:
print("Skipping {path}, not in import_paths".format(
path=filepath))
skipped = True
if skip_paths:
for skip_path in skip_paths:
if filepath.lower().startswith('{}/{}'.format(
args.export_directory, skip_path)):
print(
"Skipping {path}, in skip_paths".format(path=filepath))
skipped = True
if skipped:
if os.path.isdir(filepath) and len(os.listdir(filepath)):
logger.warn('{path} contains additional content that will be '
'skipped.'.format(path=filepath))
return
original_path = filepath[len(args.export_directory):]
if retain_paths:
importtype = get_import_type(data, original_path, 'retain_paths')
else:
importtype = get_import_type(data, original_path)
path = importtype.get_path()
if container is None:
logger.warn(
'Skipped {} because of creation error'.format(filepath))
return
_id = path.split('/')[-1]
create = True
if _id in container.objectIds():
if args.overwrite:
existing = container[_id]
if IFolder.providedBy(existing):
if len(existing.objectIds()):
print("OVERWRITE: Deleting non-empty container {path}".
format(path=path))
else:
print("OVERWRITE: Deleting content item at {path}".format(
path=path))
api.content.delete(container[_id])
else:
create = False
creation_data = importtype.get_data()
pc_data = importtype.get_post_creation_data()
creation_data['container'] = container
aspect = ISelectableConstrainTypes(container, None)
if aspect:
if (aspect.getConstrainTypesMode() != 1 or [creation_data['type']]
!= aspect.getImmediatelyAddableTypes()):
aspect.setConstrainTypesMode(1)
aspect.setImmediatelyAddableTypes([creation_data['type']])
if create:
if ignore_uuids and '_plone.uuid' in creation_data:
del creation_data['_plone.uuid']
obj = None
if not args.overwrite and (_id in container.objectIds()):
print('Skipping {path}, already exists. Use --overwrite to'
' create anyway.'.format(path=path))
return
elif (not ignore_uuids
and api.content.get(UID=creation_data['_plone.uuid'])
is not None):
logger.warn(
'Skipping {path}, content with its UUID already exists.'
'Use --ignore-uuids to create anyway.'.format(path=path))
return
else:
try:
obj = api.content.create(safe_id=True, **creation_data)
print('Created {path}'.format(path=path))
self.imported_count += 1
if self.imported_count % 50 == 0:
print('%i processed, committing' % self.imported_count)
transaction.commit()
except api.exc.InvalidParameterError:
if stop_if_exception:
logger.error(
'Error creating content {}'.format(filepath),
exc_info=True)
if pdb_if_exception:
pdb.set_trace()
raise
logger.error('Error creating content {}'.format(filepath),
exc_info=True)
return
# TODO check default folder pages came over as folder with rich text tile
# TODO any folder pages without default page should have content listing tile
else:
obj = container[_id]
for key, value in creation_data.items():
if key not in ('id', 'type'):
setattr(obj, key, value)
if obj is not None:
if path != original_path:
storage = getUtility(IRedirectionStorage)
rpath = os.path.join('/'.join(site.getPhysicalPath()),
original_path.strip('/'))
storage.add(rpath, "/".join(obj.getPhysicalPath()))
obj.contentLayout = importtype.layout
importtype.post_creation(obj, post_creation_data=pc_data)
if not args.skip_transitioning and data['state']:
# transition item only if it needs it
state = api.content.get_state(obj=obj)
if state != data['state']:
try:
print('Transitioning %s to %s' %
(obj.id, data['state']))
api.content.transition(obj, to_state=data['state'])
except Exception:
logger.error(
"Error transitioning %s to %s, maybe workflows"
" don't match up" % (obj.id, data['state']))
# pass
if stop_if_exception:
if pdb_if_exception:
pdb.set_trace()
raise
# set workflow / review history
if 'review_history' in data:
review_history = data['review_history']
wtool = api.portal.get_tool(name='portal_workflow')
# loop over all workflow chains (usually only 1)
for workflow_id in wtool.getChainFor(obj):
obj.workflow_history[workflow_id] = review_history
else:
logger.warn('No review history on {obj}'.format(obj=obj))
fix_html_images(obj)
obj.reindexObject()
try:
modification_date = data['data']['modification_date']
obj.setModificationDate(modification_date)
obj.reindexObject(idxs=['modified'])
logger.info(' set modification date to %s' %
modification_date)
except Exception:
logger.info(
'Could not set modification date on {obj}'.format(obj=obj))
return obj
def import_object(filepath, count):
fi = open(filepath)
data = mjson.loads(fi.read())
fi.close()
if filepath.endswith('__folder__'):
filepath = '/'.join(filepath.split('/')[:-1])
if data['portal_type'] in ('Topic', 'Collection', 'PressRoom'):
return
original_path = filepath[len(args.export_directory):]
importtype = get_import_type(data, original_path, _read_phase)
path = importtype.get_path()
folder = recursive_create_path('/'.join(path.split('/')[:-1]))
if folder is None:
print('Skipped {} because of creation error'.format(filepath))
_id = path.split('/')[-1]
create = True
if _id in folder.objectIds():
if args.overwrite:
if data['portal_type'] == 'Folder':
cnt = folder[_id]
if len(folder.objectIds()) == 1 or len(cnt.objectIds()) == 0:
api.content.delete(folder[_id])
else:
api.content.delete(folder[_id])
else:
create = False
print('import path: %s' % path)
creation_data = importtype.get_data()
creation_data['container'] = folder
aspect = ISelectableConstrainTypes(folder, None)
if aspect:
if (aspect.getConstrainTypesMode() != 1 or
[creation_data['type']] != aspect.getImmediatelyAddableTypes()):
aspect.setConstrainTypesMode(1)
aspect.setImmediatelyAddableTypes([creation_data['type']])
if create:
try:
obj = api.content.create(**creation_data)
except api.exc.InvalidParameterError:
return logger.error('Error creating content {}'.format(filepath),
exc_info=True)
else:
obj = folder[_id]
for key, value in creation_data.items():
if key not in ('id', 'type'):
setattr(obj, key, value)
if path != original_path:
storage = getUtility(IRedirectionStorage)
rpath = os.path.join('/'.join(site.getPhysicalPath()),
original_path.strip('/'))
storage.add(rpath, "/".join(obj.getPhysicalPath()))
importtype.post_creation(obj)
if data['state']:
try:
api.content.transition(obj, to_state=data['state'])
except:
# maybe workflows do not match up
pass
fix_html_images(obj)
obj.reindexObject()
if count % 50 == 0:
print('%i processed, committing' % count)
transaction.commit()
app._p_jar.invalidateCache() # noqa
transaction.begin()
app._p_jar.sync() # noqa