def index_fields(fields, context):
items = []
for name, field in fields:
value = getattr(context, name)
if getattr(field, 'vocabularyName', None):
if type(value) in [ListType, TupleType]:
vals = [
context._vocabulary_value(field.vocabularyName, v)
for v in value
]
else:
vals = context._vocabulary_value(field.vocabularyName, value)
items.extend(to_unicode(vals))
if IRichTextValue.providedBy(value):
html = value.output
transforms = api.portal.get_tool('portal_transforms')
if isinstance(html, unicode):
html = html.encode('utf-8')
value = transforms.convertTo(
'text/plain', html, mimetype='text/html').getData().strip()
if value:
items.extend(to_unicode(value))
return items
def text(self):
if not getattr(self.context, 'text', False):
return
if not IRichTextValue.providedBy(self.context.text):
return
util = Html2Amp()
return util(self.context.text.raw)
def get_attrs(self, html_compatibility, source_language):
schema = get_dx_schema(self.context)
attrs = []
for key in self.attrs:
if key.startswith('qSEO_'):
if self.context.hasProperty(key):
value = self.context.getProperty(key)
else:
continue
else:
field = schema[key]
if ILanguageIndependentField.providedBy(field):
logger.warn(
"Exporting language independent attribute %s, "
"this may give unexpected results during import such as all "
"language versions have the value of the last language set "
"in the attribute!" % key)
value = field.get(self.context)
if IRichTextValue.providedBy(value):
value = value.raw
if isinstance(value, unicode):
value = value.encode('UTF-8')
if not value:
value = ''
data = dict(id=key, value=value, source_language=source_language)
if html_compatibility:
attrs.append(HTML_ATTR_BODY % data)
else:
attrs.append(XLIFF_ATTR_BODY % data)
return attrs
def get_attrs(self, html_compatibility, source_language):
schema = get_dx_schema(self.context)
attrs = []
for key in self.attrs:
field = schema[key]
if ILanguageIndependentField.providedBy(field):
logger.warn(
"Exporting language independent attribute %s, "
"this may give unexpected results during import such as all "
"language versions have the value of the last language set "
"in the attribute!" % key
)
value = field.get(self.context)
if IRichTextValue.providedBy(value):
value = value.raw
if isinstance(value, unicode):
value = value.encode("UTF-8")
data = dict(id=key, value=value, source_language=source_language)
if html_compatibility:
attrs.append(HTML_ATTR_BODY % data)
else:
attrs.append(XLIFF_ATTR_BODY % data)
return attrs
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
return value
elif isinstance(value, unicode):
return self.field.fromUnicode(value)
elif value is None:
return None
raise ValueError("Cannot convert %s to an IRichTextValue" % repr(value))
def disclaimer(self):
item = getattr(self.context, 'disclaimer', None)
value = item and getattr(item, 'text', None)
if value:
if IRichTextValue.providedBy(value):
return value.output
else:
return value
return ""
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
return value
elif isinstance(value, six.text_type):
return self.field.fromUnicode(value)
elif value is None:
return None
raise ValueError('Can not convert {0:s} to an IRichTextValue'.format(
repr(value)))
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
return value
elif isinstance(value, unicode):
return self.field.fromUnicode(value)
elif value is None:
return None
raise ValueError("Cannot convert %s to an IRichTextValue" %
repr(value))
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
return value
elif isinstance(value, six.text_type):
return self.field.fromUnicode(value)
elif value is None:
return None
raise ValueError(
'Can not convert {0:s} to an IRichTextValue'.format(repr(value))
)
def toFieldValue(self, value):
if IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
elif isinstance(value, unicode):
if value == u'':
return self.field.missing_value
return self.field.fromUnicode(value)
raise ValueError("Cannot convert %s to an IRichTextValue" % repr(value))
def toFieldValue(self, value):
if IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
elif isinstance(value, unicode):
if value == u'':
return self.field.missing_value
return self.field.fromUnicode(value)
raise ValueError("Cannot convert %s to an IRichTextValue" %
repr(value))
def getValue(self, name):
value = getattr(self, name, None)
if name == "html" and IRichTextValue.providedBy(value):
if value.mimeType == value.outputMimeType:
return self.raw_encoded
else:
transformer = ITransformer(self.context, None)
if transformer is None:
return None
return transformer(value, value.outputMimeType)
return value
def _expand_binary_data(self, obj, data):
max_size = es_config.max_blobsize
is_archetype = False
if HAS_ARCHETYPES and IBaseContent.providedBy(obj):
is_archetype = True
schema = obj.Schema()
for fieldname in self._iterate_binary_fields(obj, data):
if fieldname not in data:
data[fieldname] = None
continue
if is_archetype:
field = schema[fieldname]
value = field.get(obj)
if value is None:
data[fieldname] = None
continue
data[fieldname + '_meta'] = data[fieldname]
if IBlobWrapper.providedBy(value):
if max_size and value.get_size() > max_size:
data[fieldname] = None
del data[fieldname + '_meta']
msg = 'File too big for ElasticSearch Indexing: {0}'
logger.info(msg.format(obj.absolute_url(), ), )
continue
with value.getBlob().open() as fh:
data[fieldname] = base64.b64encode(fh.read())
elif ITextField.providedBy(field):
data[fieldname] = base64.b64encode(
data[fieldname + '_meta']['data'].encode('utf8'))
else:
field = getattr(obj, fieldname, None)
if field is None:
data[fieldname] = None
continue
data[fieldname + '_meta'] = data[fieldname]
if IBlobby.providedBy(field):
if max_size and field.getSize() > max_size:
data[fieldname] = None
del data[fieldname + '_meta']
msg = 'File too big for ElasticSearch Indexing: {0}'
logger.info(msg.format(obj.absolute_url(), ), )
continue
with field.open() as fh:
data[fieldname] = base64.b64encode(fh.read())
elif IRichTextValue.providedBy(field):
data[fieldname] = base64.b64encode(
data[fieldname + '_meta']['data'].encode('utf8'), )
if max_size and len(data[fieldname]) > max_size:
data[fieldname] = None
del data[fieldname + '_meta']
logger.info(
'File too big for ElasticSearch Indexing: {0}'.format(
obj.absolute_url(), ), )
def is_richtext_value(thing):
"""Checks if the value is a richtext value
:param thing: The thing to test
:type thing: any
:returns: True if the thing is a richtext value
:rtype: bool
"""
if HAS_PLONE_APP_TEXTFIELD:
return IRichTextValue.providedBy(thing)
return False
def toFieldValue(self, value):
if IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
elif isinstance(value, six.text_type):
if value == u'':
return self.field.missing_value
return self.field.fromUnicode(value)
raise ValueError('Can not convert {0:s} to an IRichTextValue'.format(
repr(value)))
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
if self.widget.mode in ('input', 'hidden'):
return value.raw
elif self.widget.mode == 'display':
return value.output_relative_to(self.field.context)
if isinstance(value, six.text_type):
return value
elif value is None:
return None
raise ValueError('Can not convert {0:s} to six.text_type'.format(
repr(value)))
def toFieldValue(self, value):
if IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
elif isinstance(value, six.text_type):
if value == u'':
return self.field.missing_value
return self.field.fromUnicode(value)
raise ValueError(
'Can not convert {0:s} to an IRichTextValue'.format(repr(value))
)
def toWidgetValue(self, value):
if IRichTextValue.providedBy(value):
if self.widget.mode in ('input', 'hidden'):
return value.raw
elif self.widget.mode == 'display':
return value.output_relative_to(self.field.context)
if isinstance(value, six.text_type):
return value
elif value is None:
return None
raise ValueError(
'Can not convert {0:s} to six.text_type'.format(repr(value))
)
def _expand_binary_data(self, obj, data):
for fieldname in self._iterate_binary_fields(obj, data):
if fieldname not in data:
continue
field = getattr(obj, fieldname, None)
if field is None:
continue
data[fieldname + '_meta'] = data[fieldname]
if IBlobby.providedBy(field):
with field.open() as fh:
data[fieldname] = base64.b64encode(fh.read())
elif IRichTextValue.providedBy(field):
data[fieldname] = base64.b64encode(
data[fieldname + '_meta']['data'].encode('utf8'), )
def toFieldValue(self, value):
if value == u'':
return self.field.missing_value
elif isinstance(value, six.text_type):
return RichTextValue(raw=value,
mimeType=self.field.default_mime_type,
outputMimeType=self.field.output_mime_type,
encoding='utf-8')
elif IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
raise ValueError('Can not convert {0:s} to an IRichTextValue'.format(
repr(value)))
def __call__(self):
if IFolder.providedBy(self.context):
self.folder = self.context
self.text = self.context.Description()
else:
self.folder = aq_parent(self.context)
value = getattr(self.context, 'text', None)
if value:
if IRichTextValue.providedBy(value):
self.text = value.output
else:
self.text = value
else:
self.text = self.context.Description()
return self.index()
def __call__(self):
if IFolder.providedBy(self.context):
self.folder = self.context
self.text = self.context.Description()
else:
self.folder = aq_parent(self.context)
value = getattr(self.context, 'text', None)
if value:
if IRichTextValue.providedBy(value):
self.text = value.output
else:
self.text = value
else:
self.text = self.context.Description()
return self.index()
def toFieldValue(self, value):
if value == u'':
return self.field.missing_value
elif isinstance(value, six.text_type):
return RichTextValue(
raw=value,
mimeType=self.field.default_mime_type,
outputMimeType=self.field.output_mime_type,
encoding='utf-8'
)
elif IRichTextValue.providedBy(value):
if value.raw == u'':
return self.field.missing_value
return value
raise ValueError(
'Can not convert {0:s} to an IRichTextValue'.format(repr(value))
)
def _expand_binary_data(self, obj, data):
max_size = es_config.max_blobsize
for fieldname in self._iterate_binary_fields(obj, data):
if fieldname not in data:
data[fieldname] = None
continue
field = getattr(obj, fieldname, None)
if field is None:
data[fieldname] = None
continue
data[fieldname + '_meta'] = data[fieldname]
if IBlobby.providedBy(field):
with field.open() as fh:
data[fieldname] = base64.b64encode(fh.read())
elif IRichTextValue.providedBy(field):
data[fieldname] = base64.b64encode(
data[fieldname + '_meta']['data'].encode('utf8'), )
if max_size and len(data[fieldname]) > max_size:
data[fieldname] = None
del data[fieldname + '_meta']
logger.info(
'File too big for ElasticSearch Indexing: {0}'.format(
obj.absolute_url(), ), )
def alert_text_normalize(text):
"""Transform text in a way that is suitable for string comparison.
:param text: text that will be transformed.
:type text: str or unicode
:returns: text normalized.
:rtype: unicode
"""
if IRichTextValue.providedBy(text):
text = text.raw
if isinstance(text, str):
text = text.decode('latin-1')
text = NBSP_RE.sub(' ', text)
parser = HTMLParser.HTMLParser()
text = parser.unescape(text)
text = text.lower()
text = u''.join(
[c
for c in unicodedata.normalize('NFKD', text)
if not unicodedata.combining(c)]
)
return text
def index_fields(fields, context):
items = []
for name, field in fields:
value = getattr(context, name)
if getattr(field, 'vocabularyName', None):
if type(value) in [ListType, TupleType]:
vals = [context._vocabulary_value(field.vocabularyName, v) for v in value]
else:
vals = context._vocabulary_value(field.vocabularyName, value)
items.extend(to_unicode(vals))
if IRichTextValue.providedBy(value):
html = value.output
transforms = api.portal.get_tool('portal_transforms')
if isinstance(html, unicode):
html = html.encode('utf-8')
value = transforms.convertTo('text/plain',
html, mimetype='text/html'
).getData().strip()
if value:
items.extend(to_unicode(value))
return items
def getNodeTextFromHtmlID(self, html_id):
name = html_id[13:]
value = self._ce_fields[name].get(self.context)
if IRichTextValue.providedBy(value):
return value.raw
return value
def dexterity_update(obj, request=None):
"""
Utility method to update the fields of all the schemas of the Dexterity
object 'obj'.
"""
modified = defaultdict(list)
if not request:
request = obj.REQUEST
# Call processInputs to decode strings to unicode, otherwise the
# z3c.form dataconverters complain.
processInputs(request)
errors = []
for schema in get_dexterity_schemas(context=obj):
for name in getFieldNames(schema):
# Only update fields which are included in the form
# Look specifically for the empty-marker used to mark
# empty checkboxes
if name not in request.form and \
'%s-empty-marker' % name not in request.form:
continue
field = schema[name]
autoform_widgets = schema.queryTaggedValue(WIDGETS_KEY, default={})
if name in autoform_widgets:
widget = autoform_widgets[name]
if not isinstance(widget, ParameterizedWidget):
widget = ParameterizedWidget(widget)
widget = widget(field, request)
else:
widget = component.getMultiAdapter((field, request),
IFieldWidget)
widget.context = obj
value = field.missing_value
widget.update()
try:
raw = widget.extract()
except MultipleErrors, e:
errors.append(e)
log.warn("Multiple errors while extracting field: %s" % name)
continue
if raw is NOT_CHANGED:
continue
if raw is NO_VALUE:
continue
if (IRichTextValue.providedBy(raw)
and api.portal.get_registry_record(
'ploneintranet.workspace.sanitize_html')):
sanitized = RichTextValue(raw=sanitize_html(
safe_unicode(raw.raw)),
mimeType=raw.mimeType,
outputMimeType=raw.outputMimeType)
if sanitized.raw != raw.raw:
log.info(
'The HTML content of field "{}" on {} was sanitised.'.
format( # noqa
name, obj.absolute_url()))
raw = sanitized
if isinstance(field, TextLine):
# textLines must not contain line breaks (field contraint)
# to prevent a ConstraintNotSatisfied error in `toFieldValue`
raw = u" ".join(safe_unicode(raw).splitlines())
try:
value = IDataConverter(widget).toFieldValue(safe_unicode(raw))
except Exception, exc:
log.warn("Error in converting value for %s: %s (%s)" %
(name, raw, str(exc)))
raise exc
try:
field.validate(value)
except interfaces.RequiredMissing, e:
errors.append(e)
log.warn("Required field have missing value: %s" % name)
def export_file(result, header_mapping, request=None):
if not result:
return None
if request is None:
request = getRequest()
transforms = api.portal.get_tool('portal_transforms')
catalog = api.portal.get_tool("portal_catalog")
data = []
for row in result:
items_dict = dict()
if getattr(row, 'getObject', None):
obj = row.getObject()
else:
obj = row
for d in header_mapping:
fieldid = d['field']
if obj is None:
items_dict[d['header']] = row(fieldid)
continue
if fieldid == '_path':
path = obj.getPhysicalPath()
virtual_path = request.physicalPathToVirtualPath(path)
items_dict[d['header']] = (
'/'.join(virtual_path)).decode('utf-8')
continue
elif fieldid == '_url':
items_dict[d['header']] = (obj.absolute_url()).decode('utf-8')
continue
value = ""
for schemata in iterSchemata(obj):
if fieldid not in schemata:
continue
field = schemata[fieldid]
try:
value = field.get(schemata(obj))
except AttributeError:
continue
if value is field.missing_value:
continue
if IRichTextValue.providedBy(value):
# Convert to plain text
value = transforms.convertTo(target_mimetype='text/plain',
orig=value.raw_encoded,
encoding=value.encoding,
mimetype=value.mimeType)
value = safe_unicode(value.getData())
break
if IList.providedBy(field):
# Check if items in list point to objects and use titles
# instead
names = []
for item_id in value:
# Loop through choices
query = dict(id=item_id)
results = catalog(**query)
if results:
names.append(results[0].Title)
if names:
value = names
value = ', '.join(value)
break
if INamed.providedBy(value):
# Image, file field
value = u'{0}/@@download/{1}'.format(row.getURL(), fieldid)
break
serializer = ISerializer(field)
value = serializer(value, {})
break
# Added due to an ascii error related to csv export. We convert to
# unicode as unicodecsv requires unicode
value = safe_unicode(value)
items_dict[d['header']] = value
data.append(items_dict)
dataset = tablib.Dataset()
dataset.dict = data
return dataset
def get_export_data(
self,
portal_type,
blob_format,
richtext_format,
blacklist,
whitelist,
query,
):
"""Return a list of dicts with a dict for each object.
The key is the name of the field/value and the value the value.
"""
all_fields = get_schema_info(portal_type, blacklist, whitelist)
results = []
catalog = api.portal.get_tool('portal_catalog')
if not query:
query = dict()
query['portal_type'] = portal_type
if 'Language' not in query and HAS_MULTILINGUAL and \
'Language' in catalog.indexes():
query['Language'] = 'all'
brains = catalog(query)
for brain in brains:
obj = brain.getObject()
item_dict = dict()
for fieldname, field in all_fields:
if fieldname in self.ADDITIONAL_MAPPING:
# The way to access the value from this fields is
# overridden in ADDITIONAL_MAPPING
continue
value = field.get(field.interface(obj))
if not value:
# set a value anyway to keep the dimensions of all
value = ''
# make sure we do no more transforms
field = None
if IRichTextValue.providedBy(value):
value = transform_richtext(value, mimetype=richtext_format)
if IRelationList.providedBy(field):
rel_val = []
for relation in value:
rel_val.append(get_url_for_relation(relation))
value = pretty_join(rel_val)
if IRelationChoice.providedBy(field):
value = get_url_for_relation(value)
if INamed.providedBy(value):
value = get_blob_url(value, brain, blob_format, fieldname)
if IDatetime.providedBy(field) or IDate.providedBy(field):
value = api.portal.get_localized_time(
value, long_format=True)
if safe_callable(value):
value = value()
if isinstance(value, list) or isinstance(value, tuple):
value = pretty_join(value)
item_dict[fieldname] = value
# Update the data with additional info or overridden getters
item_dict.update(self.additional_data(obj, blacklist))
results.append(item_dict)
return results
def get_export_data(
self,
portal_type,
blob_format,
richtext_format,
blacklist,
whitelist,
query,
):
"""Return a list of dicts with a dict for each object.
The key is the name of the field/value and the value the value.
"""
all_fields = get_schema_info(portal_type, blacklist, whitelist)
results = []
catalog = api.portal.get_tool('portal_catalog')
if not query:
query = dict()
query['portal_type'] = portal_type
if 'Language' not in query and HAS_MULTILINGUAL and \
'Language' in catalog.indexes():
query['Language'] = 'all'
if 'path' not in query:
query['path'] = {}
query['path']['query'] = '/'.join(self.context.getPhysicalPath())
brains = catalog(query)
for brain in brains:
obj = brain.getObject()
item_dict = dict()
for fieldname, field in all_fields:
if fieldname in self.ADDITIONAL_MAPPING:
# The way to access the value from this fields is
# overridden in ADDITIONAL_MAPPING
continue
value = field.get(field.interface(obj))
if not value:
# set a value anyway to keep the dimensions of all
value = ''
# make sure we do no more transforms
field = None
if IRichTextValue.providedBy(value):
value = transform_richtext(value, mimetype=richtext_format)
if IRelationList.providedBy(field):
rel_val = []
for relation in value:
rel_val.append(get_url_for_relation(relation))
value = pretty_join(rel_val)
if IRelationChoice.providedBy(field):
value = get_url_for_relation(value)
if INamed.providedBy(value):
value = get_blob_url(value, brain, blob_format, fieldname)
if IDatetime.providedBy(field) or IDate.providedBy(field):
value = api.portal.get_localized_time(value,
long_format=True)
if safe_callable(value):
value = value()
if isinstance(value, list) or isinstance(value, tuple):
value = pretty_join(value)
if HAS_GEOLOCATION and isinstance(value, Geolocation):
value = value.__dict__
item_dict[fieldname] = value
# Update the data with additional info or overridden getters
item_dict.update(self.additional_data(obj, blacklist))
results.append(item_dict)
return results
def getNodeTextFromHtmlID(self, html_id):
name = html_id[13:]
value = self._ce_fields[name].get(self.context)
if IRichTextValue.providedBy(value):
return value.raw
return value
def get_export_data(
self,
portal_type,
blob_format,
richtext_format,
blacklist,
whitelist,
query,
):
"""Return a list of dicts with a dict for each object.
The key is the name of the field/value and the value the value.
"""
all_fields = get_schema_info(portal_type, blacklist, whitelist)
results = []
catalog = api.portal.get_tool('portal_catalog')
if not query:
query = dict()
query['portal_type'] = portal_type
if 'Language' not in query and HAS_MULTILINGUAL and \
'Language' in catalog.indexes():
query['Language'] = 'all'
brains = catalog.unrestrictedSearchResults(query)
for brain in brains:
obj = brain.getObject()
item_dict = dict()
for fieldname, field in all_fields:
if fieldname in self.ADDITIONAL_MAPPING:
# The way to access the value from this fields is
# overridden in ADDITIONAL_MAPPING
continue
try:
value = field.get(field.interface(obj))
except:
print("Skipping object at {0}".format(obj.absolute_url()))
break
if not value:
# set a value anyway to keep the dimensions of all
value = ''
# make sure we do no more transforms
field = None
if IRichTextValue.providedBy(value):
value = transform_richtext(value, mimetype=richtext_format)
if IRelationList.providedBy(field):
rel_val = []
for relation in value:
rel_val.append(get_url_for_relation(relation))
value = pretty_join(rel_val)
if IRelationChoice.providedBy(field):
value = get_url_for_relation(value)
if INamed.providedBy(value):
value = get_blob_url(value, brain, blob_format, fieldname)
if ICollection.providedBy(field):
r = []
for v in value:
if INamed.providedBy(v):
r.append(u'{0}/@@download/{1}'.format(
obj.absolute_url(), fieldname))
# r.append(base64.b64encode(v.data))
else:
r.append(v)
value = r
if IDatetime.providedBy(field) or IDate.providedBy(field):
if value.year < 1000:
if value.year < 16:
year = value.year + 2000
else:
year = value.year + 1900
if IDate.providedBy(field):
value = datetime.date(month=value.month,
day=value.day,
year=year)
elif IDatetime.providedBy(field):
value = datetime.datetime(month=value.month,
day=value.day,
year=year,
hour=value.hour,
minute=value.minute,
second=value.second)
value = api.portal.get_localized_time(value,
long_format=True)
if safe_callable(value):
value = value()
if isinstance(value, list) or isinstance(value, tuple):
value = pretty_join(value)
item_dict[fieldname] = value
else:
# Update the data with additional info or overridden getters
item_dict.update(self.additional_data(obj, blacklist))
results.append(item_dict)
continue # executed if the loop ended normally (no break)
break # executed if 'continue' was skipped (break)
return results