def copy_items(items, source, target, target_user, target_folder=None,
relationships=None, work_dir=tempfile.gettempdir()):
""" Copy items from the source portal to the target portal."""
if not target.is_logged_in():
raise PortalError('Must be logged into target portal to copy')
# Make sure the folder exists (or gets created) on the target portal
target_folder_id = None
if target_folder:
target_folder_id = _get_or_create_folder(target, target_user,
target_folder)
# Create a temporary folder to use for copying items.
copy_dir = tempfile.mkdtemp(prefix='copy_items_',
dir=unicode_to_ascii(work_dir))
try:
# Copy the items
copied_items = _copy_items(items, source, target, target_user,
target_folder_id, None, copy_dir)
# Copy the related items (if specified)
if relationships:
related_items = _copy_relationships(copied_items, source, target,
target_user, target_folder_id, relationships,
copy_dir)
copied_items.update(related_items)
finally:
if clean_temp_files:
shutil.rmtree(copy_dir)
return copied_items
def serialize_items(self, items, path, portal=None):
""" Serialize items to JSON. """
base_dir = os.path.abspath(path)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
elif not os.path.isdir(base_dir):
base_dir = os.path.dirname(base_dir)
for item in items:
item_dir = os.path.join(base_dir, item['id'])
if not os.path.exists(item_dir):
os.makedirs(item_dir)
# Write the thumbnail to a file (per the name specified in the item)
if self.thumbnails:
if not portal:
raise PortalError('The "portal" argument is required to '\
+ 'download thumbnails')
thumbnail = item.get('thumbnail')
if thumbnail:
portal.item_thumbnaild(item['id'], item_dir, thumbnail)
# Handle the data
if self.data:
if not portal:
raise PortalError('The "portal" argument is required to '\
+ 'download data')
if item['type'] in TEXT_BASED_ITEM_TYPES:
text = portal.item_data(item['id'])
if text and len(text) > 0:
item['text'] = text
elif item['type'] in FILE_BASED_ITEM_TYPES:
data_dir = os.path.join(item_dir, 'data')
if not os.path.exists(data_dir):
os.makedirs(data_dir)
portal.item_datad(item['id'], data_dir, item.get('name'))
# Write the metadata to a file
if self.metadata:
if not portal:
raise PortalError('The "portal" argument is required to '\
+ 'download metadata')
portal.item_metadatad(item['id'], item_dir)
# Write the item itself to a file (do this at the end, as the data
# will get writen to the item if the item type is text)
self.to_file(item, os.path.join(item_dir, 'item.json'))
def deserialize_users(self, path):
""" Deserialize users from CSV. """
users = []
if not os.path.isfile(path):
raise PortalError('Specific path is not a file: ' + path)
user_reader = csv.DictReader(open(path, "rb"))
for user in user_reader:
users.append(user)
return users
def deserialize_groups(self, path):
""" Deerialize groups from CSV. """
groups = []
if not os.path.isfile(path):
raise PortalError('Specific path is not a file: ' + path)
group_reader = csv.DictReader(open(path, "rb"))
for group in group_reader:
groups.append(group)
return groups
def copy_user_contents(source, source_user, target, target_user, ids=None,
relationships=None, work_dir=tempfile.gettempdir()):
""" Copy a user's items from the source portal to the target portal."""
if not source.is_logged_in():
raise PortalError('Must be logged into source portal to copy a '\
+ 'user\'s contents')
if not target.is_logged_in():
raise PortalError('Must be logged into target portal to copy a '\
+ 'user\'s contents')
# Get the user's content
root_items, folders = source.user_contents(source_user)
# Create a temporary folder to use for copying items.
copy_dir = tempfile.mkdtemp(prefix='copy_user_content_',
dir=unicode_to_ascii(work_dir))
try:
# Copy the items in the root folder
copied_items = _copy_items(root_items, source, target, target_user, None,
ids, copy_dir)
# Loop over all of the folders in the source portal, and get or create
# the corresponding folder in the target portal
for folder_id, folder_title, items in folders:
target_folder_id = _get_or_create_folder(target, target_user, folder_title)
copied_folder_items = _copy_items(items, source, target, target_user,
target_folder_id, ids, copy_dir)
copied_items.update(copied_folder_items)
# Copy the related items (if specified)
if relationships:
related_items = _copy_relationships(copied_items, source, target,
target_user, None, relationships, copy_dir)
copied_items.update(related_items)
finally:
if clean_temp_files:
shutil.rmtree(copy_dir)
return copied_items
def create_basemap_gallery_group(portal,
title,
desc=None,
snippet=None,
tags='Basemap',
phone=None,
access='org',
invitation_only=True,
thumbnail=None,
copy=True,
copy_filter=None):
# If it's a single tenant portal change 'org' access to 'public' access
if not portal.is_multitenant() and access == 'org':
access = 'public'
# Prepare the group object
group = {
'title': title,
'tags': tags,
'access': access,
'isinvitationonly': invitation_only
}
if desc:
group['description'] = desc
if snippet:
group['snippet'] = snippet
if phone:
group['phone'] = phone
# Create the group
group_id = portal.create_group(group, thumbnail)
if not group_id:
raise PortalError('Unable to create basemap group: ' + title)
# Share the contents of the current basemap group, if directed to do so
if copy:
old_group_id = _prop_to_group_id(portal, 'basemapGalleryGroupQuery')
if old_group_id:
item_query = 'group:' + old_group_id
if copy_filter:
item_query += ' ' + copy_filter
item_ids = unpack(
portal.search(['id'], item_query, scope='public'))
for item_id in item_ids:
portal.share_item(item_id, [group_id])
# Update the portal to use the new basemap gallery group
portal.update_property('basemapGalleryGroupQuery', 'id:' + group_id)
return group_id
def load_users(portal, path, f='json', cls=None, **kw):
""" Load users stored on disk into the portal. """
if portal.is_multitenant():
raise PortalError('Loading users into a multi-tenant portal is not '
+ 'supported at this time')
# Deserialize the users, and then loop over them one at a time to
# add them to the portal
deserializer = _select_deserializer(f, cls, **kw)
users = deserializer.deserialize_users(path)
for user in users:
# Remove any properties that have no entry
for property in list(user.keys()):
if user[property] is None:
del user[property]
# Signup users in the portal using the signup operation
portal.signup(user['username'], user['password'],
user['fullname'], user.get('email'))
return users
def serialize_groups(self, groups, path, portal=None):
""" Serialize groups to CSV. """
groups_copy = copy.deepcopy(groups)
field_names = GROUP_EXTRACT_PROPERTIES
if self.thumbnails:
if not portal:
raise PortalError('The "portal" argument is required to '\
+ 'download thumbnails')
field_names.append('thumbnail')
base_dir = os.path.dirname(path)
for i, group in enumerate(groups):
if 'thumbnail' in group:
group_dir = os.path.join(base_dir, group['id'])
thumbnail_path = portal.group_thumbnaild(
group['id'], group_dir, group['thumbnail'])
groups_copy[i]['thumbnail'] = os.path.relpath(
thumbnail_path, base_dir)
group_writer = csv.DictWriter(open(path, "wb"), field_names)
group_writer.writeheader()
group_writer.writerows(groups_copy)
def serialize_groups(self, groups, path, portal=None):
""" Serialize groups to JSON. """
base_dir = os.path.abspath(path)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
elif not os.path.isdir(base_dir):
base_dir = os.path.dirname(base_dir)
for group in groups:
group_dir = os.path.join(base_dir, group['id'])
if not os.path.exists(group_dir):
os.makedirs(group_dir)
# Write the thumbnail to a file (per the name specified in the item)
if self.thumbnails:
if not portal:
raise PortalError('The "portal" argument is required to '\
+ 'download thumbnails')
thumbnail = group.get('thumbnail')
if thumbnail:
portal.item_thumbnaild(group['id'], group_dir, thumbnail)
# Write the group itself to a file
self.to_file(group, os.path.join(group_dir, 'group.json'))
def _select_serializer(f, cls, **kw):
if not cls:
cls = _known_serializers.get(f)
if not cls:
raise PortalError('Unsupported format \'' + f + '\' for serialization')
return cls(**kw)
def copy_groups(groups, source, target, target_owner=None,
work_dir=tempfile.gettempdir()):
""" Copy group from the source portal to the target portal."""
if not target.is_logged_in():
raise PortalError('Must be logged into target portal to copy')
# Create the temporary directory to use for copying groups (required
# for thumbnails)
copy_dir = tempfile.mkdtemp(prefix='copy_groups_', dir=work_dir)
# Loop over each of the groups, copying one at a time
copied_groups = dict()
try:
for group in groups:
groupid = group['id']
group_dir = os.path.join(copy_dir, groupid)
os.makedirs(group_dir)
# Create a new groups with the subset of properties we want to
# copy to the target portal. Handle switching between org and
# public access when going from an org in a multitenant portal
# and a single tenant portal
target_group = _select_properties(group, GROUP_COPY_PROPERTIES)
if target_group['access'] == 'org'\
and not target.is_multitenant():
target_group['access'] = 'public'
elif target_group['access'] == 'public'\
and not source.is_multitenant()\
and target.is_multitenant() and target.is_org():
target_group['access'] = 'org'
# Handle the thumbnail (if one exists)
thumbnail_file = None
if 'thumbnail' in group:
thumbnail_file = source.group_thumbnaild(
groupid, group_dir, group['thumbnail'])
# Create the group in the target portal
target_groupid = target.create_group(target_group, thumbnail_file)
# If the group was created successfully, handling reassigning
# the group (if the target_owner is specified and it's
# different from the logged in user of the target portal
if target_groupid:
target_username = target.logged_in_user()['username']
if target_owner and (target_owner != target_username):
target.reassign_group(target_groupid, target_owner)
target.leave_group(target_groupid)
copied_groups[groupid] = target_groupid
_log.info('Copied group ' + groupid + ' in source portal '
+ 'to ' + target_groupid + ' in target portal')
else:
_log.warning('Group ' + groupid + ' was not copied '\
+ 'to target portal')
# Clean up the group directories as we go
if clean_temp_files:
shutil.rmtree(group_dir)
# Make sure we clean up the whole copy folder
finally:
if clean_temp_files:
shutil.rmtree(copy_dir)
return copied_groups