def update(self):
catalog = getToolByName(self.context, 'portal_catalog')
count = 0
for brain in catalog():
try:
obj = brain.getObject()
except (AttributeError, NotFound, KeyError):
msg = 'Catalog inconsistency: {0} not found!'
logger.error(msg.format(brain.getPath()), exc_info=1)
continue
try:
modifiedContent(obj, 'dummy event parameter')
count += 1
except Exception:
msg = 'Error updating linkintegrity-info for {0}.'
logger.error(msg.format(obj.absolute_url()), exc_info=1)
if count % 1000 == 0:
savepoint(optimistic=True)
return count
def update(self):
catalog = getToolByName(self.context, 'portal_catalog')
count = 0
query = {}
if HAS_MULTILINGUAL and 'Language' in catalog.indexes():
query['Language'] = 'all'
for brain in catalog(query):
try:
obj = brain.getObject()
except (AttributeError, NotFound, KeyError):
msg = "Catalog inconsistency: {} not found!"
logger.error(msg.format(brain.getPath()), exc_info=1)
continue
try:
modifiedContent(obj, 'dummy event parameter')
count += 1
except Exception:
msg = "Error updating linkintegrity-info for {}."
logger.error(msg.format(obj.absolute_url()), exc_info=1)
return count
def update(self):
catalog = getToolByName(self.context, 'portal_catalog')
count = 0
query = {}
if HAS_MULTILINGUAL and 'Language' in catalog.indexes():
query['Language'] = 'all'
for brain in catalog(query):
try:
obj = brain.getObject()
except (AttributeError, NotFound, KeyError):
msg = "Catalog inconsistency: {} not found!"
logger.error(msg.format(brain.getPath()), exc_info=1)
continue
try:
modifiedContent(obj, 'dummy event parameter')
count += 1
except Exception:
msg = "Error updating linkintegrity-info for {}."
logger.error(msg.format(obj.absolute_url()), exc_info=1)
if count % 1000 == 0:
savepoint(optimistic=True)
return count
def __call__(self, force=False):
if not self.data.data:
raise Exception("No data provided.")
parent = self.parent
if not parent:
raise Exception("Cannot find parent object for %s" % self.path)
_id = self.getId()
if not self.exists:
# People have no title or description.
if self.portal_type in ('agsci_person', ):
item = createContentInContainer(parent,
self.portal_type,
id=_id,
checkConstraints=False)
else:
item = createContentInContainer(
parent,
self.portal_type,
id=_id,
title=self.data.title,
description=self.data.description,
checkConstraints=False)
# Set UID
setattr(item, ATTRIBUTE_NAME, self.UID)
else:
# If the item exists, and it's published, no further changes
if self.review_state in [
'published',
] and not force:
return
item = self.context
# Set subject (tags)
if self.data.subject:
item.setSubject(list(self.data.subject))
# Set HTML
html = self.html
if html:
item.text = RichTextValue(raw=html,
mimeType=u'text/html',
outputMimeType='text/x-html-safe')
# Set File field
file = self.file
if file:
item.file = self.file
# Set Lead Image or Image field
image = self.image
if image:
item.image = image
# Unset full width field if image is too small, or is portrait.
try:
(w, h) = image.getImageSize()
except:
pass
else:
if w < h or w < 600:
item.image_full_width = False
# Set field values
# Map current field name 'field' to old 'data_field' from feed.
fields = self.fields
field_names = self.field_names
for field_name in field_names:
field = fields.get(field_name)
if self.map_fields:
data_field = self.fields_mapping.get(field_name, field_name)
else:
data_field = field_name
# Skip fields if we're only importing specific fields
if self.include_fields and field_name not in self.include_fields:
continue
if field_name not in self.exclude_fields:
value = getattr(self.data, data_field, None)
if value or isinstance(value, (bool, )):
value = self.transform_value(
field=field,
field_name=data_field,
value=value,
)
setattr(item, field_name, value)
if self.debug:
LOG(
self.__class__.__name__, INFO,
"%s: Setting %s to %r" %
(item.absolute_url(), field_name, value))
# Set collection criteria
if self.portal_type in ('Collection', 'Newsletter'):
if self.data.collection_criteria:
item.setQuery(self.data.collection_criteria)
if self.data.collection_sort_field:
item.setSort_on(self.data.collection_sort_field)
if self.data.collection_sort_reversed:
item.setSort_reversed(True)
# Set default page
if self.data.default_page:
default_page_id = safe_unicode(
self.data.default_page).encode('utf-8')
self.context.setDefaultPage(default_page_id)
else:
# Set layout if no default page
layout = self.data.layout
if layout in self.valid_layouts:
item.setLayout(layout)
# Set dates
effective = self.data.effective_date
expires = self.data.expiration_date
if effective:
item.setEffectiveDate(DateTime(effective))
if expires:
item.setExpirationDate(DateTime(expires))
# If event, set start and end
if self.portal_type in ('Event', ):
start_date = localize(DateTime(self.data.start_date))
end_date = localize(DateTime(self.data.end_date))
acc = IEventAccessor(item)
acc.start = start_date
acc.end = end_date
# Set references
modifiedContent(item, None)
# Reindex
item.reindexObject()
def link_objects(source, target, relationship):
"""Create a relation from source to target using zc.relation
For RelationChoice or RelationList it will add the relation as attribute.
Other relations they will only be added to the relation-catalog.
"""
if not IDexterityContent.providedBy(source):
logger.info(u'{} is no dexterity content'.format(source.portal_type))
return
if not IDexterityContent.providedBy(target):
logger.info(u'{} is no dexterity content'.format(target.portal_type))
return
relation_catalog = getUtility(ICatalog)
intids = getUtility(IIntIds)
to_id = intids.getId(target)
from_id = intids.getId(source)
from_attribute = relationship
# Check if there is exactly this relation.
# If so remove it and create a fresh one.
query = {
'from_attribute': from_attribute,
'from_id': from_id,
'to_id': to_id,
}
for rel in relation_catalog.findRelations(query):
relation_catalog.unindex(rel)
if from_attribute == referencedRelationship:
# Don't mess with linkintegrity-relations!
# Refresh them by triggering this subscriber.
modifiedContent(source, None)
return
if from_attribute == ITERATE_RELATION_NAME:
# Iterate relations use a subclass of RelationValue
relation = StagingRelationValue(to_id)
event._setRelation(source, ITERATE_RELATION_NAME, relation)
return
field_and_schema = get_field_and_schema_for_fieldname(
from_attribute, source.portal_type)
if field_and_schema is None:
# The relationship is not the name of a field. Only create a relation.
logger.info(u'No field. Setting relation {} from {} to {}'.format(
source.absolute_url(), target.absolute_url(), relationship))
event._setRelation(source, from_attribute, RelationValue(to_id))
return
field, schema = field_and_schema
if isinstance(field, RelationList):
logger.info('Add relation to relationlist {} from {} to {}'.format(
from_attribute, source.absolute_url(), target.absolute_url()))
existing_relations = getattr(source, from_attribute, [])
existing_relations.append(RelationValue(to_id))
setattr(source, from_attribute, existing_relations)
modified(source)
return
elif isinstance(field, (Relation, RelationChoice)):
logger.info('Add relation {} from {} to {}'.format(
from_attribute, source.absolute_url(), target.absolute_url()))
setattr(source, from_attribute, RelationValue(to_id))
modified(source)
return
# We should never end up here!
logger.info('Warning: Unexpected relation {} from {} to {}'.format(
from_attribute, source.absolute_url(), target.absolute_url()))
def restore_relations(context=None, all_relations=None):
"""Restore relations from a annotation on the portal.
"""
portal = api.portal.get()
if all_relations is None:
all_relations = IAnnotations(portal)[RELATIONS_KEY]
logger.info('Loaded {0} relations to restore'.format(len(all_relations)))
update_linkintegrity = set()
modified_items = set()
modified_relation_lists = defaultdict(list)
# remove duplicates but keep original order
unique_relations = []
seen = set()
seen_add = seen.add
for i in all_relations:
hashable = tuple(i.items())
if hashable not in seen:
unique_relations.append(i)
seen_add(hashable)
else:
logger.info(u'Dropping duplicate: {}'.format(hashable))
if len(unique_relations) < len(all_relations):
logger.info('Dropping {0} duplicates'.format(
len(all_relations) - len(unique_relations)))
all_relations = unique_relations
intids = getUtility(IIntIds)
for index, item in enumerate(all_relations, start=1):
if not index % 500:
logger.info(u'Restored {} of {} relations...'.format(
index, len(all_relations)))
source_obj = uuidToObject(item['from_uuid'])
target_obj = uuidToObject(item['to_uuid'])
if not source_obj:
logger.info(u'{} is missing'.format(item['from_uuid']))
continue
if not target_obj:
logger.info(u'{} is missing'.format(item['to_uuid']))
continue
if not IDexterityContent.providedBy(source_obj):
logger.info(u'{} is no dexterity content'.format(
source_obj.portal_type))
continue
if not IDexterityContent.providedBy(target_obj):
logger.info(u'{} is no dexterity content'.format(
target_obj.portal_type))
continue
from_attribute = item['from_attribute']
to_id = intids.getId(target_obj)
if from_attribute == referencedRelationship:
# Ignore linkintegrity for now. We'll rebuilt it at the end!
update_linkintegrity.add(item['from_uuid'])
continue
if from_attribute == ITERATE_RELATION_NAME:
# Iterate relations are not set as values of fields
relation = StagingRelationValue(to_id)
event._setRelation(source_obj, ITERATE_RELATION_NAME, relation)
continue
field_and_schema = get_field_and_schema_for_fieldname(
from_attribute, source_obj.portal_type)
if field_and_schema is None:
# the from_attribute is no field
# we could either create a fresh relation or log the case
logger.info(u'No field. Setting relation: {}'.format(item))
event._setRelation(source_obj, from_attribute,
RelationValue(to_id))
continue
field, schema = field_and_schema
relation = RelationValue(to_id)
if isinstance(field, RelationList):
logger.info(
'Add relation to relationslist {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
if item['from_uuid'] in modified_relation_lists.get(
from_attribute, []):
# Do not purge relations
existing_relations = getattr(source_obj, from_attribute, [])
else:
# First touch. Make sure we purge!
existing_relations = []
existing_relations.append(relation)
setattr(source_obj, from_attribute, existing_relations)
modified_items.add(item['from_uuid'])
modified_relation_lists[from_attribute].append(item['from_uuid'])
continue
elif isinstance(field, (Relation, RelationChoice)):
logger.info('Add relation {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
setattr(source_obj, from_attribute, relation)
modified_items.add(item['from_uuid'])
continue
else:
# we should never end up here!
logger.info('Warning: Unexpected relation {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
update_linkintegrity = set(update_linkintegrity)
logger.info('Updating linkintegrity for {} items'.format(
len(update_linkintegrity)))
for uuid in sorted(update_linkintegrity):
modifiedContent(uuidToObject(uuid), None)
logger.info('Updating relations for {} items'.format(len(modified_items)))
for uuid in sorted(modified_items):
obj = uuidToObject(uuid)
# updateRelations from z3c.relationfield does not properly update relations in behaviors
# that are registered with a marker-interface.
# update_behavior_relations (from plone.app.relationfield) does that but does not update
# those in the main schema. Duh!
updateRelations(obj, None)
update_behavior_relations(obj, None)
# purge annotation from portal if they exist
if RELATIONS_KEY in IAnnotations(portal):
del IAnnotations(portal)[RELATIONS_KEY]
logger.info('Done!')
def restore_relations(context=None, all_relations=None):
"""Restore relations from a annotation on the portal.
"""
portal = api.portal.get()
if all_relations is None:
all_relations = IAnnotations(portal)[RELATIONS_KEY]
logger.info('Loaded {0} relations to restore'.format(len(all_relations)))
update_linkintegrity = []
modified_items = []
modified_relation_lists = defaultdict(list)
# remove duplicates but keep original order
seen = set()
seen_add = seen.add
unique_relations = [
i for i in all_relations
if not (tuple(i.items()) in seen or seen_add(tuple(i.items())))
]
if len(unique_relations) < len(all_relations):
logger.info('Dropping {0} duplicates'.format(
len(all_relations) - len(unique_relations)))
all_relations = unique_relations
intids = getUtility(IIntIds)
for item in all_relations:
source_obj = uuidToObject(item['from_uuid'])
target_obj = uuidToObject(item['to_uuid'])
if not source_obj:
logger.info(u'{} is missing'.format(item['from_uuid']))
continue
if not target_obj:
logger.info(u'{} is missing'.format(item['to_uuid']))
continue
if not IDexterityContent.providedBy(source_obj):
logger.info(u'{} is no dexterity content'.format(
source_obj.portal_type))
continue
if not IDexterityContent.providedBy(target_obj):
logger.info(u'{} is no dexterity content'.format(
target_obj.portal_type))
continue
from_attribute = item['from_attribute']
to_id = intids.getId(target_obj)
if from_attribute == referencedRelationship:
# Ignore linkintegrity for now. We'll rebuilt it at the end!
update_linkintegrity.append(item['from_uuid'])
continue
if from_attribute == ITERATE_RELATION_NAME:
# Iterate relations are not set as values of fields
relation = StagingRelationValue(to_id)
event._setRelation(source_obj, ITERATE_RELATION_NAME, relation)
continue
fti = getUtility(IDexterityFTI, name=source_obj.portal_type)
field_and_schema = get_field_and_schema_for_fieldname(
from_attribute, fti)
if field_and_schema is None:
# the from_attribute is no field
# we could either create a fresh relation or log the case
logger.info(u'No field. Setting relation: {}'.format(item))
event._setRelation(source_obj, from_attribute,
RelationValue(to_id))
continue
field, schema = field_and_schema
relation = RelationValue(to_id)
if isinstance(field, RelationList):
logger.info(
'Add relation to relationslist {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
if item['from_uuid'] in modified_relation_lists.get(
from_attribute, []):
# Do not purge relations
existing_relations = getattr(source_obj, from_attribute, [])
else:
# First touch. Make sure we purge!
existing_relations = []
existing_relations.append(relation)
setattr(source_obj, from_attribute, existing_relations)
modified_items.append(item['from_uuid'])
modified_relation_lists[from_attribute].append(item['from_uuid'])
continue
elif isinstance(field, (Relation, RelationChoice)):
logger.info('Add relation {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
setattr(source_obj, from_attribute, relation)
modified_items.append(item['from_uuid'])
continue
else:
# we should never end up here!
logger.info('Warning: Unexpected relation {} from {} to {}'.format(
from_attribute, source_obj.absolute_url(),
target_obj.absolute_url()))
to_update = set(update_linkintegrity + modified_items)
if to_update:
logger.info('Reindexing {} items'.format(len(to_update)))
for uuid in sorted(to_update):
# call modified for all changed items
modifiedContent(uuidToObject(uuid), None)
# purge annotation from portal if they exist
if RELATIONS_KEY in IAnnotations(portal):
del IAnnotations(portal)[RELATIONS_KEY]