def index_fields(fields, context): items = [] for name, field in fields: value = getattr(context, name) if getattr(field, 'vocabularyName', None): if type(value) in [ListType, TupleType]: vals = [ context._vocabulary_value(field.vocabularyName, v) for v in value ] else: vals = context._vocabulary_value(field.vocabularyName, value) items.extend(to_unicode(vals)) if IRichTextValue.providedBy(value): html = value.output transforms = api.portal.get_tool('portal_transforms') if isinstance(html, unicode): html = html.encode('utf-8') value = transforms.convertTo( 'text/plain', html, mimetype='text/html').getData().strip() if value: items.extend(to_unicode(value)) return items
def text(self): if not getattr(self.context, 'text', False): return if not IRichTextValue.providedBy(self.context.text): return util = Html2Amp() return util(self.context.text.raw)
def get_attrs(self, html_compatibility, source_language): schema = get_dx_schema(self.context) attrs = [] for key in self.attrs: if key.startswith('qSEO_'): if self.context.hasProperty(key): value = self.context.getProperty(key) else: continue else: field = schema[key] if ILanguageIndependentField.providedBy(field): logger.warn( "Exporting language independent attribute %s, " "this may give unexpected results during import such as all " "language versions have the value of the last language set " "in the attribute!" % key) value = field.get(self.context) if IRichTextValue.providedBy(value): value = value.raw if isinstance(value, unicode): value = value.encode('UTF-8') if not value: value = '' data = dict(id=key, value=value, source_language=source_language) if html_compatibility: attrs.append(HTML_ATTR_BODY % data) else: attrs.append(XLIFF_ATTR_BODY % data) return attrs
def get_attrs(self, html_compatibility, source_language): schema = get_dx_schema(self.context) attrs = [] for key in self.attrs: field = schema[key] if ILanguageIndependentField.providedBy(field): logger.warn( "Exporting language independent attribute %s, " "this may give unexpected results during import such as all " "language versions have the value of the last language set " "in the attribute!" % key ) value = field.get(self.context) if IRichTextValue.providedBy(value): value = value.raw if isinstance(value, unicode): value = value.encode("UTF-8") data = dict(id=key, value=value, source_language=source_language) if html_compatibility: attrs.append(HTML_ATTR_BODY % data) else: attrs.append(XLIFF_ATTR_BODY % data) return attrs
def toWidgetValue(self, value): if IRichTextValue.providedBy(value): return value elif isinstance(value, unicode): return self.field.fromUnicode(value) elif value is None: return None raise ValueError("Cannot convert %s to an IRichTextValue" % repr(value))
def disclaimer(self): item = getattr(self.context, 'disclaimer', None) value = item and getattr(item, 'text', None) if value: if IRichTextValue.providedBy(value): return value.output else: return value return ""
def toWidgetValue(self, value): if IRichTextValue.providedBy(value): return value elif isinstance(value, six.text_type): return self.field.fromUnicode(value) elif value is None: return None raise ValueError('Can not convert {0:s} to an IRichTextValue'.format( repr(value)))
def toWidgetValue(self, value): if IRichTextValue.providedBy(value): return value elif isinstance(value, six.text_type): return self.field.fromUnicode(value) elif value is None: return None raise ValueError( 'Can not convert {0:s} to an IRichTextValue'.format(repr(value)) )
def toFieldValue(self, value): if IRichTextValue.providedBy(value): if value.raw == u'': return self.field.missing_value return value elif isinstance(value, unicode): if value == u'': return self.field.missing_value return self.field.fromUnicode(value) raise ValueError("Cannot convert %s to an IRichTextValue" % repr(value))
def getValue(self, name): value = getattr(self, name, None) if name == "html" and IRichTextValue.providedBy(value): if value.mimeType == value.outputMimeType: return self.raw_encoded else: transformer = ITransformer(self.context, None) if transformer is None: return None return transformer(value, value.outputMimeType) return value
def _expand_binary_data(self, obj, data): max_size = es_config.max_blobsize is_archetype = False if HAS_ARCHETYPES and IBaseContent.providedBy(obj): is_archetype = True schema = obj.Schema() for fieldname in self._iterate_binary_fields(obj, data): if fieldname not in data: data[fieldname] = None continue if is_archetype: field = schema[fieldname] value = field.get(obj) if value is None: data[fieldname] = None continue data[fieldname + '_meta'] = data[fieldname] if IBlobWrapper.providedBy(value): if max_size and value.get_size() > max_size: data[fieldname] = None del data[fieldname + '_meta'] msg = 'File too big for ElasticSearch Indexing: {0}' logger.info(msg.format(obj.absolute_url(), ), ) continue with value.getBlob().open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif ITextField.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8')) else: field = getattr(obj, fieldname, None) if field is None: data[fieldname] = None continue data[fieldname + '_meta'] = data[fieldname] if IBlobby.providedBy(field): if max_size and field.getSize() > max_size: data[fieldname] = None del data[fieldname + '_meta'] msg = 'File too big for ElasticSearch Indexing: {0}' logger.info(msg.format(obj.absolute_url(), ), ) continue with field.open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif IRichTextValue.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8'), ) if max_size and len(data[fieldname]) > max_size: data[fieldname] = None del data[fieldname + '_meta'] logger.info( 'File too big for ElasticSearch Indexing: {0}'.format( obj.absolute_url(), ), )
def is_richtext_value(thing): """Checks if the value is a richtext value :param thing: The thing to test :type thing: any :returns: True if the thing is a richtext value :rtype: bool """ if HAS_PLONE_APP_TEXTFIELD: return IRichTextValue.providedBy(thing) return False
def toFieldValue(self, value): if IRichTextValue.providedBy(value): if value.raw == u'': return self.field.missing_value return value elif isinstance(value, six.text_type): if value == u'': return self.field.missing_value return self.field.fromUnicode(value) raise ValueError('Can not convert {0:s} to an IRichTextValue'.format( repr(value)))
def toWidgetValue(self, value): if IRichTextValue.providedBy(value): if self.widget.mode in ('input', 'hidden'): return value.raw elif self.widget.mode == 'display': return value.output_relative_to(self.field.context) if isinstance(value, six.text_type): return value elif value is None: return None raise ValueError('Can not convert {0:s} to six.text_type'.format( repr(value)))
def toFieldValue(self, value): if IRichTextValue.providedBy(value): if value.raw == u'': return self.field.missing_value return value elif isinstance(value, six.text_type): if value == u'': return self.field.missing_value return self.field.fromUnicode(value) raise ValueError( 'Can not convert {0:s} to an IRichTextValue'.format(repr(value)) )
def toWidgetValue(self, value): if IRichTextValue.providedBy(value): if self.widget.mode in ('input', 'hidden'): return value.raw elif self.widget.mode == 'display': return value.output_relative_to(self.field.context) if isinstance(value, six.text_type): return value elif value is None: return None raise ValueError( 'Can not convert {0:s} to six.text_type'.format(repr(value)) )
def _expand_binary_data(self, obj, data): for fieldname in self._iterate_binary_fields(obj, data): if fieldname not in data: continue field = getattr(obj, fieldname, None) if field is None: continue data[fieldname + '_meta'] = data[fieldname] if IBlobby.providedBy(field): with field.open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif IRichTextValue.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8'), )
def toFieldValue(self, value): if value == u'': return self.field.missing_value elif isinstance(value, six.text_type): return RichTextValue(raw=value, mimeType=self.field.default_mime_type, outputMimeType=self.field.output_mime_type, encoding='utf-8') elif IRichTextValue.providedBy(value): if value.raw == u'': return self.field.missing_value return value raise ValueError('Can not convert {0:s} to an IRichTextValue'.format( repr(value)))
def __call__(self): if IFolder.providedBy(self.context): self.folder = self.context self.text = self.context.Description() else: self.folder = aq_parent(self.context) value = getattr(self.context, 'text', None) if value: if IRichTextValue.providedBy(value): self.text = value.output else: self.text = value else: self.text = self.context.Description() return self.index()
def toFieldValue(self, value): if value == u'': return self.field.missing_value elif isinstance(value, six.text_type): return RichTextValue( raw=value, mimeType=self.field.default_mime_type, outputMimeType=self.field.output_mime_type, encoding='utf-8' ) elif IRichTextValue.providedBy(value): if value.raw == u'': return self.field.missing_value return value raise ValueError( 'Can not convert {0:s} to an IRichTextValue'.format(repr(value)) )
def _expand_binary_data(self, obj, data): max_size = es_config.max_blobsize for fieldname in self._iterate_binary_fields(obj, data): if fieldname not in data: data[fieldname] = None continue field = getattr(obj, fieldname, None) if field is None: data[fieldname] = None continue data[fieldname + '_meta'] = data[fieldname] if IBlobby.providedBy(field): with field.open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif IRichTextValue.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8'), ) if max_size and len(data[fieldname]) > max_size: data[fieldname] = None del data[fieldname + '_meta'] logger.info( 'File too big for ElasticSearch Indexing: {0}'.format( obj.absolute_url(), ), )
def alert_text_normalize(text): """Transform text in a way that is suitable for string comparison. :param text: text that will be transformed. :type text: str or unicode :returns: text normalized. :rtype: unicode """ if IRichTextValue.providedBy(text): text = text.raw if isinstance(text, str): text = text.decode('latin-1') text = NBSP_RE.sub(' ', text) parser = HTMLParser.HTMLParser() text = parser.unescape(text) text = text.lower() text = u''.join( [c for c in unicodedata.normalize('NFKD', text) if not unicodedata.combining(c)] ) return text
def index_fields(fields, context): items = [] for name, field in fields: value = getattr(context, name) if getattr(field, 'vocabularyName', None): if type(value) in [ListType, TupleType]: vals = [context._vocabulary_value(field.vocabularyName, v) for v in value] else: vals = context._vocabulary_value(field.vocabularyName, value) items.extend(to_unicode(vals)) if IRichTextValue.providedBy(value): html = value.output transforms = api.portal.get_tool('portal_transforms') if isinstance(html, unicode): html = html.encode('utf-8') value = transforms.convertTo('text/plain', html, mimetype='text/html' ).getData().strip() if value: items.extend(to_unicode(value)) return items
def getNodeTextFromHtmlID(self, html_id): name = html_id[13:] value = self._ce_fields[name].get(self.context) if IRichTextValue.providedBy(value): return value.raw return value
def dexterity_update(obj, request=None): """ Utility method to update the fields of all the schemas of the Dexterity object 'obj'. """ modified = defaultdict(list) if not request: request = obj.REQUEST # Call processInputs to decode strings to unicode, otherwise the # z3c.form dataconverters complain. processInputs(request) errors = [] for schema in get_dexterity_schemas(context=obj): for name in getFieldNames(schema): # Only update fields which are included in the form # Look specifically for the empty-marker used to mark # empty checkboxes if name not in request.form and \ '%s-empty-marker' % name not in request.form: continue field = schema[name] autoform_widgets = schema.queryTaggedValue(WIDGETS_KEY, default={}) if name in autoform_widgets: widget = autoform_widgets[name] if not isinstance(widget, ParameterizedWidget): widget = ParameterizedWidget(widget) widget = widget(field, request) else: widget = component.getMultiAdapter((field, request), IFieldWidget) widget.context = obj value = field.missing_value widget.update() try: raw = widget.extract() except MultipleErrors, e: errors.append(e) log.warn("Multiple errors while extracting field: %s" % name) continue if raw is NOT_CHANGED: continue if raw is NO_VALUE: continue if (IRichTextValue.providedBy(raw) and api.portal.get_registry_record( 'ploneintranet.workspace.sanitize_html')): sanitized = RichTextValue(raw=sanitize_html( safe_unicode(raw.raw)), mimeType=raw.mimeType, outputMimeType=raw.outputMimeType) if sanitized.raw != raw.raw: log.info( 'The HTML content of field "{}" on {} was sanitised.'. format( # noqa name, obj.absolute_url())) raw = sanitized if isinstance(field, TextLine): # textLines must not contain line breaks (field contraint) # to prevent a ConstraintNotSatisfied error in `toFieldValue` raw = u" ".join(safe_unicode(raw).splitlines()) try: value = IDataConverter(widget).toFieldValue(safe_unicode(raw)) except Exception, exc: log.warn("Error in converting value for %s: %s (%s)" % (name, raw, str(exc))) raise exc try: field.validate(value) except interfaces.RequiredMissing, e: errors.append(e) log.warn("Required field have missing value: %s" % name)
def export_file(result, header_mapping, request=None): if not result: return None if request is None: request = getRequest() transforms = api.portal.get_tool('portal_transforms') catalog = api.portal.get_tool("portal_catalog") data = [] for row in result: items_dict = dict() if getattr(row, 'getObject', None): obj = row.getObject() else: obj = row for d in header_mapping: fieldid = d['field'] if obj is None: items_dict[d['header']] = row(fieldid) continue if fieldid == '_path': path = obj.getPhysicalPath() virtual_path = request.physicalPathToVirtualPath(path) items_dict[d['header']] = ( '/'.join(virtual_path)).decode('utf-8') continue elif fieldid == '_url': items_dict[d['header']] = (obj.absolute_url()).decode('utf-8') continue value = "" for schemata in iterSchemata(obj): if fieldid not in schemata: continue field = schemata[fieldid] try: value = field.get(schemata(obj)) except AttributeError: continue if value is field.missing_value: continue if IRichTextValue.providedBy(value): # Convert to plain text value = transforms.convertTo(target_mimetype='text/plain', orig=value.raw_encoded, encoding=value.encoding, mimetype=value.mimeType) value = safe_unicode(value.getData()) break if IList.providedBy(field): # Check if items in list point to objects and use titles # instead names = [] for item_id in value: # Loop through choices query = dict(id=item_id) results = catalog(**query) if results: names.append(results[0].Title) if names: value = names value = ', '.join(value) break if INamed.providedBy(value): # Image, file field value = u'{0}/@@download/{1}'.format(row.getURL(), fieldid) break serializer = ISerializer(field) value = serializer(value, {}) break # Added due to an ascii error related to csv export. We convert to # unicode as unicodecsv requires unicode value = safe_unicode(value) items_dict[d['header']] = value data.append(items_dict) dataset = tablib.Dataset() dataset.dict = data return dataset
def get_export_data( self, portal_type, blob_format, richtext_format, blacklist, whitelist, query, ): """Return a list of dicts with a dict for each object. The key is the name of the field/value and the value the value. """ all_fields = get_schema_info(portal_type, blacklist, whitelist) results = [] catalog = api.portal.get_tool('portal_catalog') if not query: query = dict() query['portal_type'] = portal_type if 'Language' not in query and HAS_MULTILINGUAL and \ 'Language' in catalog.indexes(): query['Language'] = 'all' brains = catalog(query) for brain in brains: obj = brain.getObject() item_dict = dict() for fieldname, field in all_fields: if fieldname in self.ADDITIONAL_MAPPING: # The way to access the value from this fields is # overridden in ADDITIONAL_MAPPING continue value = field.get(field.interface(obj)) if not value: # set a value anyway to keep the dimensions of all value = '' # make sure we do no more transforms field = None if IRichTextValue.providedBy(value): value = transform_richtext(value, mimetype=richtext_format) if IRelationList.providedBy(field): rel_val = [] for relation in value: rel_val.append(get_url_for_relation(relation)) value = pretty_join(rel_val) if IRelationChoice.providedBy(field): value = get_url_for_relation(value) if INamed.providedBy(value): value = get_blob_url(value, brain, blob_format, fieldname) if IDatetime.providedBy(field) or IDate.providedBy(field): value = api.portal.get_localized_time( value, long_format=True) if safe_callable(value): value = value() if isinstance(value, list) or isinstance(value, tuple): value = pretty_join(value) item_dict[fieldname] = value # Update the data with additional info or overridden getters item_dict.update(self.additional_data(obj, blacklist)) results.append(item_dict) return results
def get_export_data( self, portal_type, blob_format, richtext_format, blacklist, whitelist, query, ): """Return a list of dicts with a dict for each object. The key is the name of the field/value and the value the value. """ all_fields = get_schema_info(portal_type, blacklist, whitelist) results = [] catalog = api.portal.get_tool('portal_catalog') if not query: query = dict() query['portal_type'] = portal_type if 'Language' not in query and HAS_MULTILINGUAL and \ 'Language' in catalog.indexes(): query['Language'] = 'all' if 'path' not in query: query['path'] = {} query['path']['query'] = '/'.join(self.context.getPhysicalPath()) brains = catalog(query) for brain in brains: obj = brain.getObject() item_dict = dict() for fieldname, field in all_fields: if fieldname in self.ADDITIONAL_MAPPING: # The way to access the value from this fields is # overridden in ADDITIONAL_MAPPING continue value = field.get(field.interface(obj)) if not value: # set a value anyway to keep the dimensions of all value = '' # make sure we do no more transforms field = None if IRichTextValue.providedBy(value): value = transform_richtext(value, mimetype=richtext_format) if IRelationList.providedBy(field): rel_val = [] for relation in value: rel_val.append(get_url_for_relation(relation)) value = pretty_join(rel_val) if IRelationChoice.providedBy(field): value = get_url_for_relation(value) if INamed.providedBy(value): value = get_blob_url(value, brain, blob_format, fieldname) if IDatetime.providedBy(field) or IDate.providedBy(field): value = api.portal.get_localized_time(value, long_format=True) if safe_callable(value): value = value() if isinstance(value, list) or isinstance(value, tuple): value = pretty_join(value) if HAS_GEOLOCATION and isinstance(value, Geolocation): value = value.__dict__ item_dict[fieldname] = value # Update the data with additional info or overridden getters item_dict.update(self.additional_data(obj, blacklist)) results.append(item_dict) return results
def get_export_data( self, portal_type, blob_format, richtext_format, blacklist, whitelist, query, ): """Return a list of dicts with a dict for each object. The key is the name of the field/value and the value the value. """ all_fields = get_schema_info(portal_type, blacklist, whitelist) results = [] catalog = api.portal.get_tool('portal_catalog') if not query: query = dict() query['portal_type'] = portal_type if 'Language' not in query and HAS_MULTILINGUAL and \ 'Language' in catalog.indexes(): query['Language'] = 'all' brains = catalog.unrestrictedSearchResults(query) for brain in brains: obj = brain.getObject() item_dict = dict() for fieldname, field in all_fields: if fieldname in self.ADDITIONAL_MAPPING: # The way to access the value from this fields is # overridden in ADDITIONAL_MAPPING continue try: value = field.get(field.interface(obj)) except: print("Skipping object at {0}".format(obj.absolute_url())) break if not value: # set a value anyway to keep the dimensions of all value = '' # make sure we do no more transforms field = None if IRichTextValue.providedBy(value): value = transform_richtext(value, mimetype=richtext_format) if IRelationList.providedBy(field): rel_val = [] for relation in value: rel_val.append(get_url_for_relation(relation)) value = pretty_join(rel_val) if IRelationChoice.providedBy(field): value = get_url_for_relation(value) if INamed.providedBy(value): value = get_blob_url(value, brain, blob_format, fieldname) if ICollection.providedBy(field): r = [] for v in value: if INamed.providedBy(v): r.append(u'{0}/@@download/{1}'.format( obj.absolute_url(), fieldname)) # r.append(base64.b64encode(v.data)) else: r.append(v) value = r if IDatetime.providedBy(field) or IDate.providedBy(field): if value.year < 1000: if value.year < 16: year = value.year + 2000 else: year = value.year + 1900 if IDate.providedBy(field): value = datetime.date(month=value.month, day=value.day, year=year) elif IDatetime.providedBy(field): value = datetime.datetime(month=value.month, day=value.day, year=year, hour=value.hour, minute=value.minute, second=value.second) value = api.portal.get_localized_time(value, long_format=True) if safe_callable(value): value = value() if isinstance(value, list) or isinstance(value, tuple): value = pretty_join(value) item_dict[fieldname] = value else: # Update the data with additional info or overridden getters item_dict.update(self.additional_data(obj, blacklist)) results.append(item_dict) continue # executed if the loop ended normally (no break) break # executed if 'continue' was skipped (break) return results