def _store_data(self, appstruct: dict): """Store data appstruct. `comments_count` value is converted from int to :class:`Btrees.Length`, to support ZODB conflict resultion. """ if self._count_field_name in appstruct: # pragma: no branch data = getattr(self.context, self._annotation_key, {}) if self._count_field_name not in data: counter = Length(0) else: counter = data[self._count_field_name] count = appstruct[self._count_field_name] counter.set(count) appstruct[self._count_field_name] = counter super()._store_data(appstruct)
def chooseName(self, name, object): context = removeAllProxies(self.context) next = getattr(context, '_next_id', None) if next is None: next = Length(0) context._next_id = next i = next() name = unicode(name.strip()) orig = name while (not name) or (name in self.context): i += 1 name = '%s%0.2d'%(orig, i) next.set(i+1) self.checkName(name, object) return name
class FormSaveDataAdapter(FormActionAdapter): """A form action adapter that will save form input data and return it in csv- or tab-delimited format.""" schema = FormAdapterSchema.copy() + Schema(( LinesField('showFields', required=0, searchable=0, vocabulary='allFieldDisplayList', widget=PicklistWidget( label=_(u'label_savefields_text', default=u"Saved Fields"), description=_(u'help_savefields_text', default=u""" Pick the fields whose inputs you'd like to include in the saved data. If empty, all fields will be saved. """), ), ), LinesField('ExtraData', widget=MultiSelectionWidget( label=_(u'label_savedataextra_text', default='Extra Data'), description=_(u'help_savedataextra_text', default=u""" Pick any extra data you'd like saved with the form input. """), format='checkbox', ), vocabulary='vocabExtraDataDL', ), StringField('DownloadFormat', searchable=0, required=1, default='csv', vocabulary='vocabFormatDL', widget=SelectionWidget( label=_(u'label_downloadformat_text', default=u'Download Format'), ), ), BooleanField("UseColumnNames", required=False, searchable=False, widget=BooleanWidget( label=_(u'label_usecolumnnames_text', default=u"Include Column Names"), description=_(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"), ), ), ExLinesField('SavedFormInput', edit_accessor='getSavedFormInputForEdit', mutator='setSavedFormInput', searchable=0, required=0, primary=1, schemata="saved data", read_permission=DOWNLOAD_SAVED_PERMISSION, widget=TextAreaWidget( label=_(u'label_savedatainput_text', default=u"Saved Form Input"), description=_(u'help_savedatainput_text'), ), ), )) schema.moveField('execCondition', pos='bottom') meta_type = 'FormSaveDataAdapter' portal_type = 'FormSaveDataAdapter' archetype_name = 'Save Data Adapter' immediate_view = 'fg_savedata_view_p3' default_view = 'fg_savedata_view_p3' suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',) security = ClassSecurityInfo() def _migrateStorage(self): # we're going to use an LOBTree for storage. we need to # consider the possibility that self is from an # older version that uses the native Archetypes storage # or the former IOBTree (<= 1.6.0b2 ) # in the SavedFormInput field. updated = base_hasattr(self, '_inputStorage') and \ base_hasattr(self, '_inputItems') and \ base_hasattr(self, '_length') if not updated: try: saved_input = self.getSavedFormInput() except AttributeError: saved_input = [] self._inputStorage = SavedDataBTree() i = 0 self._inputItems = 0 self._length = Length() if len(saved_input): for row in saved_input: self._inputStorage[i] = row i += 1 self.SavedFormInput = [] self._inputItems = i self._length.set(i) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput') def getSavedFormInput(self): """ returns saved input as an iterable; each row is a sequence of fields. """ if base_hasattr(self, '_inputStorage'): return self._inputStorage.values() else: return self.SavedFormInput security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems') def getSavedFormInputItems(self): """ returns saved input as an iterable; each row is an (id, sequence of fields) tuple """ if base_hasattr(self, '_inputStorage'): return self._inputStorage.items() else: return enumerate(self.SavedFormInput) security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit') def getSavedFormInputForEdit(self, **kwargs): """ returns saved as CSV text """ delimiter = self.csvDelimiter() sbuf = StringIO() writer = csv.writer(sbuf, delimiter=delimiter) for row in self.getSavedFormInput(): writer.writerow(row) res = sbuf.getvalue() sbuf.close() return res security.declareProtected(ModifyPortalContent, 'setSavedFormInput') def setSavedFormInput(self, value, **kwargs): """ expects value as csv text string, stores as list of lists """ self._migrateStorage() self._inputStorage.clear() i = 0 self._inputItems = 0 self._length.set(0) if len(value): delimiter = self.csvDelimiter() sbuf = StringIO(value) reader = csv.reader(sbuf, delimiter=delimiter) for row in reader: if row: self._inputStorage[i] = row i += 1 self._inputItems = i self._length.set(i) sbuf.close() # logger.debug("setSavedFormInput: %s items" % self._inputItems) def _clearSavedFormInput(self): # convenience method to clear input buffer self._migrateStorage() self._inputStorage.clear() self._inputItems = 0 self._length.set(0) security.declareProtected(ModifyPortalContent, 'clearSavedFormInput') def clearSavedFormInput(self, **kwargs): """ clear input buffer TTW """ plone.protect.CheckAuthenticator(self.REQUEST) plone.protect.PostOnly(self.REQUEST) self._clearSavedFormInput() self.REQUEST.response.redirect(self.absolute_url()) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById') def getSavedFormInputById(self, id): """ Return the data stored for record with 'id' """ lst = [field.replace('\r', '').replace('\n', r'\n') for field in self._inputStorage[id]] return lst security.declareProtected(ModifyPortalContent, 'manage_saveData') def manage_saveData(self, id, data): """ Save the data for record with 'id' """ plone.protect.CheckAuthenticator(self.REQUEST) plone.protect.PostOnly(self.REQUEST) self._migrateStorage() lst = list() for i in range(0, len(self.getColumnNames())): lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n')) self._inputStorage[id] = lst self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view') security.declareProtected(ModifyPortalContent, 'manage_deleteData') def manage_deleteData(self, id): """ Delete the data for record with 'id' """ self._migrateStorage() del self._inputStorage[id] self._inputItems -= 1 self._length.change(-1) self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view') def _addDataRow(self, value): self._migrateStorage() if isinstance(self._inputStorage, IOBTree): # 32-bit IOBTree; use a key which is more likely to conflict # but which won't overflow the key's bits id = self._inputItems self._inputItems += 1 else: # 64-bit LOBTree id = int(time.time() * 1000) while id in self._inputStorage: # avoid collisions during testing id += 1 self._inputStorage[id] = value self._length.change(1) security.declareProtected(ModifyPortalContent, 'addDataRow') def addDataRow(self, value): # """ a wrapper for the _addDataRow method """ self._addDataRow(value) security.declarePrivate('onSuccess') def onSuccess(self, fields, REQUEST=None, loopstop=False): # """ # saves data. # """ if LP_SAVE_TO_CANONICAL and not loopstop: # LinguaPlone functionality: # check to see if we're in a translated # form folder, but not the canonical version. parent = self.aq_parent if safe_hasattr(parent, 'isTranslation') and \ parent.isTranslation() and not parent.isCanonical(): # look in the canonical version to see if there is # a matching (by id) save-data adapter. # If so, call its onSuccess method cf = parent.getCanonical() target = cf.get(self.getId()) if target is not None and target.meta_type == 'FormSaveDataAdapter': target.onSuccess(fields, REQUEST, loopstop=True) return from ZPublisher.HTTPRequest import FileUpload data = [] for f in fields: showFields = getattr(self, 'showFields', []) if showFields and f.id not in showFields: continue if f.isFileField(): file = REQUEST.form.get('%s_file' % f.fgField.getName()) if isinstance(file, FileUpload) and file.filename != '': file.seek(0) fdata = file.read() filename = file.filename mimetype, enc = guess_content_type(filename, fdata, None) if mimetype.find('text/') >= 0: # convert to native eols fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n') data.append('%s:%s:%s:%s' % (filename, mimetype, enc, fdata)) else: data.append('%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc)) else: data.append('NO UPLOAD') elif not f.isLabel(): val = REQUEST.form.get(f.fgField.getName(), '') if not type(val) in StringTypes: # Zope has marshalled the field into # something other than a string val = str(val) data.append(val) if self.ExtraData: for f in self.ExtraData: if f == 'dt': data.append(str(DateTime())) else: data.append(getattr(REQUEST, f, '')) self._addDataRow(data) security.declareProtected(ModifyPortalContent, 'allFieldDisplayList') def allFieldDisplayList(self): # """ returns a DisplayList of all fields """ return self.fgFieldsDisplayList() security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames') def getColumnNames(self, excludeServerSide=True): # """Returns a list of column names""" showFields = getattr(self, 'showFields', []) names = [field.getName() for field in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide) if not showFields or field.getName() in showFields] for f in self.ExtraData: names.append(f) return names security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles') def getColumnTitles(self, excludeServerSide=True): # """Returns a list of column titles""" names = [field.widget.label for field in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide)] for f in self.ExtraData: names.append(self.vocabExtraDataDL().getValue(f, '')) return names def _cleanInputForTSV(self, value): # make data safe to store in tab-delimited format return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t') security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv') def download_tsv(self, REQUEST=None, RESPONSE=None): # """Download the saved data # """ filename = self.id if filename.find('.') < 0: filename = '%s.tsv' % filename header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset()) if getattr(self, 'UseColumnNames', False): res = "%s\n" % '\t'.join(self.getColumnNames(excludeServerSide=False)) if isinstance(res, unicode): res = res.encode(self.getCharset()) else: res = '' for row in self.getSavedFormInput(): res = '%s%s\n' % (res, '\t'.join([self._cleanInputForTSV(col) for col in row])) return res security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv') def download_csv(self, REQUEST=None, RESPONSE=None): # """Download the saved data # """ filename = self.id if filename.find('.') < 0: filename = '%s.csv' % filename header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset()) if getattr(self, 'UseColumnNames', False): delimiter = self.csvDelimiter() res = "%s\n" % delimiter.join(self.getColumnNames(excludeServerSide=False)) if isinstance(res, unicode): res = res.encode(self.getCharset()) else: res = '' return '%s%s' % (res, self.getSavedFormInputForEdit()) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_xls') def download_xls(self, REQUEST=None, RESPONSE=None): # """Download the saved data # """ filename = self.id if filename.find('.') < 0: filename = '%s.xls' % filename header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) RESPONSE.setHeader("Content-Type", 'application/vnd.ms-excel') xldoc = xlwt.Workbook(encoding=self.getCharset()) sheet = xldoc.add_sheet(self.Title()) row_num = 0 if getattr(self, 'UseColumnNames', False): col_names = self.getColumnNames(excludeServerSide=False) for idx, label in enumerate(col_names): sheet.write(0, idx, label.encode(self.getCharset())) row_num += 1 for row in self.getSavedFormInput(): for col_num, col in enumerate(row): if type(col) is unicode: col = col.encode(self.getCharset()) if urlparse(col).scheme in ('http', 'https'): col = xlwt.Formula('HYPERLINK("%(url)s")' % dict(url=col)) else: for format in (int, float): try: col = format(col) break except ValueError: pass sheet.write(row_num, col_num, col) row_num += 1 string_buffer = StringIO() xldoc.save(string_buffer) return string_buffer.getvalue() security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download') def download(self, REQUEST=None, RESPONSE=None): """Download the saved data """ format = getattr(self, 'DownloadFormat', 'tsv') if format == 'tsv': return self.download_tsv(REQUEST, RESPONSE) if format == 'xls': assert has_xls, 'xls download not available' return self.download_xls(REQUEST, RESPONSE) else: assert format == 'csv', 'Unknown download format' return self.download_csv(REQUEST, RESPONSE) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict') def rowAsColDict(self, row, cols): # """ Where row is a data sequence and cols is a column name sequence, # returns a dict of colname:column. This is a convenience method # used in the record view. # """ colcount = len(cols) rdict = {} for i in range(0, len(row)): if i < colcount: rdict[cols[i]] = row[i] else: rdict['column-%s' % i] = row[i] return rdict security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries') def inputAsDictionaries(self): # """returns saved data as an iterable of dictionaries # """ cols = self.getColumnNames() for row in self.getSavedFormInput(): yield self.rowAsColDict(row, cols) # alias for old mis-naming security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries') InputAsDictionaries = inputAsDictionaries security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME') def formatMIME(self): # """MIME format selected for download # """ format = getattr(self, 'DownloadFormat', 'tsv') if format == 'tsv': return 'text/tab-separated-values' if format == 'xls': return 'application/vnd.ms-excel' else: assert format == 'csv', 'Unknown download format' return 'text/comma-separated-values' security.declarePrivate('csvDelimiter') def csvDelimiter(self): # """Delimiter character for CSV downloads # """ fgt = getToolByName(self, 'formgen_tool') return fgt.getCSVDelimiter() security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved') def itemsSaved(self): # """Download the saved data # """ if base_hasattr(self, '_length'): return self._length() elif base_hasattr(self, '_inputItems'): return self._inputItems else: return len(self.SavedFormInput) def vocabExtraDataDL(self): # """ returns vocabulary for extra data """ return DisplayList(( ('dt', self.translate(msgid='vocabulary_postingdt_text', domain='ploneformgen', default='Posting Date/Time') ), ('HTTP_X_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR',), ('REMOTE_ADDR', 'REMOTE_ADDR',), ('HTTP_USER_AGENT', 'HTTP_USER_AGENT',), )) def vocabFormatDL(self): # """ returns vocabulary for format """ formats = [ ('tsv', self.translate(msgid='vocabulary_tsv_text', domain='ploneformgen', default='Tab-Separated Values') ), ('csv', self.translate(msgid='vocabulary_csv_text', domain='ploneformgen', default='Comma-Separated Values') ), ] if has_xls: formats.append( ('xls', self.translate(msgid='vocabulary_xls_doc', domain='ploneformgen', default='Excel document') ), ) return DisplayList(formats)
class CachingCatalog(Catalog): implements(ICatalog) os = os # for unit tests generation = None # b/c def __init__(self): super(CachingCatalog, self).__init__() self.generation = Length(0) def clear(self): self.invalidate() super(CachingCatalog, self).clear() def index_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).index_doc(*arg, **kw) def unindex_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).unindex_doc(*arg, **kw) def reindex_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).reindex_doc(*arg, **kw) def __setitem__(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).__setitem__(*arg, **kw) @MetricMod('CS.%s') @metricmethod def search(self, *arg, **kw): use_cache = True if 'use_cache' in kw: use_cache = kw.pop('use_cache') if 'NO_CATALOG_CACHE' in self.os.environ: use_cache = False if 'tags' in kw: # The tags index changes without invalidating the catalog, # so don't cache any query involving the tags index. use_cache = False if not use_cache: return self._search(*arg, **kw) cache = queryUtility(ICatalogSearchCache) if cache is None: return self._search(*arg, **kw) key = cPickle.dumps((arg, kw)) generation = self.generation if generation is None: generation = Length(0) genval = generation.value if (genval == 0) or (genval > cache.generation): # an update in another process requires that the local cache be # invalidated cache.clear() cache.generation = genval if cache.get(key) is None: start = time.time() num, docids = self._search(*arg, **kw) # We don't cache large result sets because the time it takes to # unroll the result set turns out to be far more time than it # takes to run the search. In a particular instance using OSI's # catalog a search that took 0.015s but returned nearly 35,295 # results took over 50s to unroll the result set for caching, # significantly slowing search performance. if num > LARGE_RESULT_SET: return num, docids # we need to unroll here; a btree-based structure may have # a reference to its connection docids = list(docids) cache[key] = (num, docids) return cache.get(key) @metricmethod def _search(self, *arg, **kw): start = time.time() res = super(CachingCatalog, self).search(*arg, **kw) duration = time.time() - start notify(CatalogQueryEvent(self, kw, duration, res)) return res def invalidate(self): # Increment the generation; this tells *another process* that # its catalog cache needs to be cleared generation = self.generation if generation is None: generation = self.generation = Length(0) if generation.value >= sys.maxint: # don't keep growing the generation integer; wrap at sys.maxint self.generation.set(0) else: self.generation.change(1) # Clear the cache for *this process* cache = queryUtility(ICatalogSearchCache) if cache is not None: cache.clear() cache.generation = self.generation.value
class BTreeFolder2Base (Persistent): """Base for BTree-based folders. """ security = ClassSecurityInfo() manage_options=( ({'label':'Contents', 'action':'manage_main',}, ) + Folder.manage_options[1:] ) security.declareProtected(view_management_screens, 'manage_main') manage_main = DTMLFile('contents', globals()) _tree = None # OOBTree: { id -> object } _count = None # A BTrees.Length _v_nextid = 0 # The integer component of the next generated ID _mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } } title = '' def __init__(self, id=None): if id is not None: self.id = id self._initBTrees() def _initBTrees(self): self._tree = OOBTree() self._count = Length() self._mt_index = OOBTree() def _populateFromFolder(self, source): """Fill this folder with the contents of another folder. """ for name in source.objectIds(): value = source._getOb(name, None) if value is not None: self._setOb(name, aq_base(value)) security.declareProtected(view_management_screens, 'manage_fixCount') def manage_fixCount(self): """Calls self._fixCount() and reports the result as text. """ old, new = self._fixCount() path = '/'.join(self.getPhysicalPath()) if old == new: return "No count mismatch detected in BTreeFolder2 at %s." % path else: return ("Fixed count mismatch in BTreeFolder2 at %s. " "Count was %d; corrected to %d" % (path, old, new)) def _fixCount(self): """Checks if the value of self._count disagrees with len(self.objectIds()). If so, corrects self._count. Returns the old and new count values. If old==new, no correction was performed. """ old = self._count() new = len(self.objectIds()) if old != new: self._count.set(new) return old, new security.declareProtected(view_management_screens, 'manage_cleanup') def manage_cleanup(self): """Calls self._cleanup() and reports the result as text. """ v = self._cleanup() path = '/'.join(self.getPhysicalPath()) if v: return "No damage detected in BTreeFolder2 at %s." % path else: return ("Fixed BTreeFolder2 at %s. " "See the log for more details." % path) def _cleanup(self): """Cleans up errors in the BTrees. Certain ZODB bugs have caused BTrees to become slightly insane. Fortunately, there is a way to clean up damaged BTrees that always seems to work: make a new BTree containing the items() of the old one. Returns 1 if no damage was detected, or 0 if damage was detected and fixed. """ from BTrees.check import check path = '/'.join(self.getPhysicalPath()) try: check(self._tree) for key in self._tree.keys(): if not self._tree.has_key(key): raise AssertionError( "Missing value for key: %s" % repr(key)) check(self._mt_index) for key, value in self._mt_index.items(): if (not self._mt_index.has_key(key) or self._mt_index[key] is not value): raise AssertionError( "Missing or incorrect meta_type index: %s" % repr(key)) check(value) for k in value.keys(): if not value.has_key(k): raise AssertionError( "Missing values for meta_type index: %s" % repr(key)) return 1 except AssertionError: LOG.warn( 'Detected damage to %s. Fixing now.' % path, exc_info=True) try: self._tree = OOBTree(self._tree) mt_index = OOBTree() for key, value in self._mt_index.items(): mt_index[key] = OIBTree(value) self._mt_index = mt_index except: LOG.error('Failed to fix %s.' % path, exc_info=True) raise else: LOG.info('Fixed %s.' % path) return 0 def _getOb(self, id, default=_marker): """Return the named object from the folder. """ tree = self._tree if default is _marker: ob = tree[id] return ob.__of__(self) else: ob = tree.get(id, _marker) if ob is _marker: return default else: return ob.__of__(self) def _setOb(self, id, object): """Store the named object in the folder. """ tree = self._tree if tree.has_key(id): raise KeyError('There is already an item named "%s".' % id) tree[id] = object self._count.change(1) # Update the meta type index. mti = self._mt_index meta_type = getattr(object, 'meta_type', None) if meta_type is not None: ids = mti.get(meta_type, None) if ids is None: ids = OIBTree() mti[meta_type] = ids ids[id] = 1 def _delOb(self, id): """Remove the named object from the folder. """ tree = self._tree meta_type = getattr(tree[id], 'meta_type', None) del tree[id] self._count.change(-1) # Update the meta type index. if meta_type is not None: mti = self._mt_index ids = mti.get(meta_type, None) if ids is not None and ids.has_key(id): del ids[id] if not ids: # Removed the last object of this meta_type. # Prune the index. del mti[meta_type] security.declareProtected(view_management_screens, 'getBatchObjectListing') def getBatchObjectListing(self, REQUEST=None): """Return a structure for a page template to show the list of objects. """ if REQUEST is None: REQUEST = {} pref_rows = int(REQUEST.get('dtpref_rows', 20)) b_start = int(REQUEST.get('b_start', 1)) b_count = int(REQUEST.get('b_count', 1000)) b_end = b_start + b_count - 1 url = self.absolute_url() + '/manage_main' idlist = self.objectIds() # Pre-sorted. count = self.objectCount() if b_end < count: next_url = url + '?b_start=%d' % (b_start + b_count) else: b_end = count next_url = '' if b_start > 1: prev_url = url + '?b_start=%d' % max(b_start - b_count, 1) else: prev_url = '' formatted = [] formatted.append(listtext0 % pref_rows) for i in range(b_start - 1, b_end): optID = escape(idlist[i]) formatted.append(listtext1 % (escape(optID, quote=1), optID)) formatted.append(listtext2) return {'b_start': b_start, 'b_end': b_end, 'prev_batch_url': prev_url, 'next_batch_url': next_url, 'formatted_list': ''.join(formatted)} security.declareProtected(view_management_screens, 'manage_object_workspace') def manage_object_workspace(self, ids=(), REQUEST=None): '''Redirects to the workspace of the first object in the list.''' if ids and REQUEST is not None: REQUEST.RESPONSE.redirect( '%s/%s/manage_workspace' % ( self.absolute_url(), quote(ids[0]))) else: return self.manage_main(self, REQUEST) security.declareProtected(access_contents_information, 'tpValues') def tpValues(self): """Ensures the items don't show up in the left pane. """ return () security.declareProtected(access_contents_information, 'objectCount') def objectCount(self): """Returns the number of items in the folder.""" return self._count() security.declareProtected(access_contents_information, 'has_key') def has_key(self, id): """Indicates whether the folder has an item by ID. """ return self._tree.has_key(id) security.declareProtected(access_contents_information, 'objectIds') def objectIds(self, spec=None): # Returns a list of subobject ids of the current object. # If 'spec' is specified, returns objects whose meta_type # matches 'spec'. if spec is not None: if isinstance(spec, StringType): spec = [spec] mti = self._mt_index set = None for meta_type in spec: ids = mti.get(meta_type, None) if ids is not None: set = union(set, ids) if set is None: return () else: return set.keys() else: return self._tree.keys() security.declareProtected(access_contents_information, 'objectValues') def objectValues(self, spec=None): # Returns a list of actual subobjects of the current object. # If 'spec' is specified, returns only objects whose meta_type # match 'spec'. return LazyMap(self._getOb, self.objectIds(spec)) security.declareProtected(access_contents_information, 'objectItems') def objectItems(self, spec=None): # Returns a list of (id, subobject) tuples of the current object. # If 'spec' is specified, returns only objects whose meta_type match # 'spec' return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)), self.objectIds(spec)) security.declareProtected(access_contents_information, 'objectMap') def objectMap(self): # Returns a tuple of mappings containing subobject meta-data. return LazyMap(lambda (k, v): {'id': k, 'meta_type': getattr(v, 'meta_type', None)}, self._tree.items(), self._count()) # superValues() looks for the _objects attribute, but the implementation # would be inefficient, so superValues() support is disabled. _objects = () security.declareProtected(access_contents_information, 'objectIds_d') def objectIds_d(self, t=None): ids = self.objectIds(t) res = {} for id in ids: res[id] = 1 return res security.declareProtected(access_contents_information, 'objectMap_d') def objectMap_d(self, t=None): return self.objectMap() def _checkId(self, id, allow_dup=0): if not allow_dup and self.has_key(id): raise BadRequestException, ('The id "%s" is invalid--' 'it is already in use.' % id) def _setObject(self, id, object, roles=None, user=None, set_owner=1, suppress_events=False): ob = object # better name, keep original function signature v = self._checkId(id) if v is not None: id = v # If an object by the given id already exists, remove it. if self.has_key(id): self._delObject(id) if not suppress_events: notify(ObjectWillBeAddedEvent(ob, self, id)) self._setOb(id, ob) ob = self._getOb(id) if set_owner: # TODO: eventify manage_fixupOwnershipAfterAdd # This will be called for a copy/clone, or a normal _setObject. ob.manage_fixupOwnershipAfterAdd() # Try to give user the local role "Owner", but only if # no local roles have been set on the object yet. if getattr(ob, '__ac_local_roles__', _marker) is None: user = getSecurityManager().getUser() if user is not None: userid = user.getId() if userid is not None: ob.manage_setLocalRoles(userid, ['Owner']) if not suppress_events: notify(ObjectAddedEvent(ob, self, id)) notifyContainerModified(self) OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self) return id def _delObject(self, id, dp=1, suppress_events=False): ob = self._getOb(id) OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self) if not suppress_events: notify(ObjectWillBeRemovedEvent(ob, self, id)) self._delOb(id) if not suppress_events: notify(ObjectRemovedEvent(ob, self, id)) notifyContainerModified(self) # Aliases for mapping-like access. __len__ = objectCount keys = objectIds values = objectValues items = objectItems # backward compatibility hasObject = has_key security.declareProtected(access_contents_information, 'get') def get(self, name, default=None): return self._getOb(name, default) # Utility for generating unique IDs. security.declareProtected(access_contents_information, 'generateId') def generateId(self, prefix='item', suffix='', rand_ceiling=999999999): """Returns an ID not used yet by this folder. The ID is unlikely to collide with other threads and clients. The IDs are sequential to optimize access to objects that are likely to have some relation. """ tree = self._tree n = self._v_nextid attempt = 0 while 1: if n % 4000 != 0 and n <= rand_ceiling: id = '%s%d%s' % (prefix, n, suffix) if not tree.has_key(id): break n = randint(1, rand_ceiling) attempt = attempt + 1 if attempt > MAX_UNIQUEID_ATTEMPTS: # Prevent denial of service raise ExhaustedUniqueIdsError self._v_nextid = n + 1 return id def __getattr__(self, name): # Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal # to subitems, and __bobo_traverse__ hooks don't work with # restrictedTraverse() unless __getattr__() is also present. # Oh well. res = self._tree.get(name) if res is None: raise AttributeError, name return res
class CachingCatalog(Catalog): implements(ICatalog) os = os # for unit tests generation = None # b/c def __init__(self): super(CachingCatalog, self).__init__() self.generation = Length(0) def clear(self): self.invalidate() super(CachingCatalog, self).clear() def index_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).index_doc(*arg, **kw) def unindex_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).unindex_doc(*arg, **kw) def reindex_doc(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).reindex_doc(*arg, **kw) def __setitem__(self, *arg, **kw): self.invalidate() super(CachingCatalog, self).__setitem__(*arg, **kw) def search(self, *arg, **kw): use_cache = True if 'use_cache' in kw: use_cache = kw.pop('use_cache') if 'NO_CATALOG_CACHE' in self.os.environ: use_cache = False if 'tags' in kw: # The tags index changes without invalidating the catalog, # so don't cache any query involving the tags index. use_cache = False if not use_cache: return self._search(*arg, **kw) cache = queryUtility(ICatalogSearchCache) if cache is None: return self._search(*arg, **kw) key = cPickle.dumps((arg, kw)) generation = self.generation if generation is None: generation = Length(0) genval = generation.value if (genval == 0) or (genval > cache.generation): # an update in another process requires that the local cache be # invalidated cache.clear() cache.generation = genval if cache.get(key) is None: num, docids = self._search(*arg, **kw) # We don't cache large result sets because the time it takes to # unroll the result set turns out to be far more time than it # takes to run the search. In a particular instance using OSI's # catalog a search that took 0.015s but returned nearly 35,295 # results took over 50s to unroll the result set for caching, # significantly slowing search performance. if num > LARGE_RESULT_SET: return num, docids # we need to unroll here; a btree-based structure may have # a reference to its connection docids = list(docids) cache[key] = (num, docids) return cache.get(key) def _search(self, *arg, **kw): start = time.time() res = super(CachingCatalog, self).search(*arg, **kw) duration = time.time() - start notify(CatalogQueryEvent(self, kw, duration, res)) return res def invalidate(self): # Increment the generation; this tells *another process* that # its catalog cache needs to be cleared generation = self.generation if generation is None: generation = self.generation = Length(0) if generation.value >= sys.maxint: # don't keep growing the generation integer; wrap at sys.maxint self.generation.set(0) else: self.generation.change(1) # Clear the cache for *this process* cache = queryUtility(ICatalogSearchCache) if cache is not None: cache.clear() cache.generation = self.generation.value
class BTreeFolder2Base(Persistent): """Base for BTree-based folders. """ security = ClassSecurityInfo() manage_options = (({ 'label': 'Contents', 'action': 'manage_main', }, ) + Folder.manage_options[1:]) security.declareProtected(view_management_screens, 'manage_main') manage_main = DTMLFile('contents', globals()) _tree = None # OOBTree: { id -> object } _count = None # A BTrees.Length _v_nextid = 0 # The integer component of the next generated ID _mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } } title = '' def __init__(self, id=None): if id is not None: self.id = id self._initBTrees() def _initBTrees(self): self._tree = OOBTree() self._count = Length() self._mt_index = OOBTree() def _populateFromFolder(self, source): """Fill this folder with the contents of another folder. """ for name in source.objectIds(): value = source._getOb(name, None) if value is not None: self._setOb(name, aq_base(value)) security.declareProtected(view_management_screens, 'manage_fixCount') def manage_fixCount(self): """Calls self._fixCount() and reports the result as text. """ old, new = self._fixCount() path = '/'.join(self.getPhysicalPath()) if old == new: return "No count mismatch detected in BTreeFolder2 at %s." % path else: return ("Fixed count mismatch in BTreeFolder2 at %s. " "Count was %d; corrected to %d" % (path, old, new)) def _fixCount(self): """Checks if the value of self._count disagrees with len(self.objectIds()). If so, corrects self._count. Returns the old and new count values. If old==new, no correction was performed. """ old = self._count() new = len(self.objectIds()) if old != new: self._count.set(new) return old, new security.declareProtected(view_management_screens, 'manage_cleanup') def manage_cleanup(self): """Calls self._cleanup() and reports the result as text. """ v = self._cleanup() path = '/'.join(self.getPhysicalPath()) if v: return "No damage detected in BTreeFolder2 at %s." % path else: return ("Fixed BTreeFolder2 at %s. " "See the log for more details." % path) def _cleanup(self): """Cleans up errors in the BTrees. Certain ZODB bugs have caused BTrees to become slightly insane. Fortunately, there is a way to clean up damaged BTrees that always seems to work: make a new BTree containing the items() of the old one. Returns 1 if no damage was detected, or 0 if damage was detected and fixed. """ from BTrees.check import check path = '/'.join(self.getPhysicalPath()) try: check(self._tree) for key in self._tree.keys(): if not self._tree.has_key(key): raise AssertionError("Missing value for key: %s" % repr(key)) check(self._mt_index) for key, value in self._mt_index.items(): if (not self._mt_index.has_key(key) or self._mt_index[key] is not value): raise AssertionError( "Missing or incorrect meta_type index: %s" % repr(key)) check(value) for k in value.keys(): if not value.has_key(k): raise AssertionError( "Missing values for meta_type index: %s" % repr(key)) return 1 except AssertionError: LOG.warn('Detected damage to %s. Fixing now.' % path, exc_info=sys.exc_info()) try: self._tree = OOBTree(self._tree) mt_index = OOBTree() for key, value in self._mt_index.items(): mt_index[key] = OIBTree(value) self._mt_index = mt_index except: LOG.error('Failed to fix %s.' % path, exc_info=sys.exc_info()) raise else: LOG.info('Fixed %s.' % path) return 0 def _getOb(self, id, default=_marker): """Return the named object from the folder. """ tree = self._tree if default is _marker: ob = tree[id] return ob.__of__(self) else: ob = tree.get(id, _marker) if ob is _marker: return default else: return ob.__of__(self) def _setOb(self, id, object): """Store the named object in the folder. """ tree = self._tree if tree.has_key(id): raise KeyError('There is already an item named "%s".' % id) tree[id] = object self._count.change(1) # Update the meta type index. mti = self._mt_index meta_type = getattr(object, 'meta_type', None) if meta_type is not None: ids = mti.get(meta_type, None) if ids is None: ids = OIBTree() mti[meta_type] = ids ids[id] = 1 def _delOb(self, id): """Remove the named object from the folder. """ tree = self._tree meta_type = getattr(tree[id], 'meta_type', None) del tree[id] self._count.change(-1) # Update the meta type index. if meta_type is not None: mti = self._mt_index ids = mti.get(meta_type, None) if ids is not None and ids.has_key(id): del ids[id] if not ids: # Removed the last object of this meta_type. # Prune the index. del mti[meta_type] security.declareProtected(view_management_screens, 'getBatchObjectListing') def getBatchObjectListing(self, REQUEST=None): """Return a structure for a page template to show the list of objects. """ if REQUEST is None: REQUEST = {} pref_rows = int(REQUEST.get('dtpref_rows', 20)) b_start = int(REQUEST.get('b_start', 1)) b_count = int(REQUEST.get('b_count', 1000)) b_end = b_start + b_count - 1 url = self.absolute_url() + '/manage_main' idlist = self.objectIds() # Pre-sorted. count = self.objectCount() if b_end < count: next_url = url + '?b_start=%d' % (b_start + b_count) else: b_end = count next_url = '' if b_start > 1: prev_url = url + '?b_start=%d' % max(b_start - b_count, 1) else: prev_url = '' formatted = [] formatted.append(listtext0 % pref_rows) for i in range(b_start - 1, b_end): optID = escape(idlist[i]) formatted.append(listtext1 % (escape(optID, quote=1), optID)) formatted.append(listtext2) return { 'b_start': b_start, 'b_end': b_end, 'prev_batch_url': prev_url, 'next_batch_url': next_url, 'formatted_list': ''.join(formatted) } security.declareProtected(view_management_screens, 'manage_object_workspace') def manage_object_workspace(self, ids=(), REQUEST=None): '''Redirects to the workspace of the first object in the list.''' if ids and REQUEST is not None: REQUEST.RESPONSE.redirect('%s/%s/manage_workspace' % (self.absolute_url(), quote(ids[0]))) else: return self.manage_main(self, REQUEST) security.declareProtected(access_contents_information, 'tpValues') def tpValues(self): """Ensures the items don't show up in the left pane. """ return () security.declareProtected(access_contents_information, 'objectCount') def objectCount(self): """Returns the number of items in the folder.""" return self._count() security.declareProtected(access_contents_information, 'has_key') def has_key(self, id): """Indicates whether the folder has an item by ID. """ return self._tree.has_key(id) security.declareProtected(access_contents_information, 'objectIds') def objectIds(self, spec=None): # Returns a list of subobject ids of the current object. # If 'spec' is specified, returns objects whose meta_type # matches 'spec'. mti = self._mt_index if spec is None: spec = mti.keys() #all meta types if isinstance(spec, StringType): spec = [spec] set = None for meta_type in spec: ids = mti.get(meta_type, None) if ids is not None: set = union(set, ids) if set is None: return () else: return set.keys() security.declareProtected(access_contents_information, 'objectValues') def objectValues(self, spec=None): # Returns a list of actual subobjects of the current object. # If 'spec' is specified, returns only objects whose meta_type # match 'spec'. return LazyMap(self._getOb, self.objectIds(spec)) security.declareProtected(access_contents_information, 'objectItems') def objectItems(self, spec=None): # Returns a list of (id, subobject) tuples of the current object. # If 'spec' is specified, returns only objects whose meta_type match # 'spec' return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)), self.objectIds(spec)) security.declareProtected(access_contents_information, 'objectMap') def objectMap(self): # Returns a tuple of mappings containing subobject meta-data. return LazyMap( lambda (k, v): { 'id': k, 'meta_type': getattr(v, 'meta_type', None) }, self._tree.items(), self._count()) # superValues() looks for the _objects attribute, but the implementation # would be inefficient, so superValues() support is disabled. _objects = () security.declareProtected(access_contents_information, 'objectIds_d') def objectIds_d(self, t=None): ids = self.objectIds(t) res = {} for id in ids: res[id] = 1 return res security.declareProtected(access_contents_information, 'objectMap_d') def objectMap_d(self, t=None): return self.objectMap() def _checkId(self, id, allow_dup=0): if not allow_dup and self.has_key(id): raise BadRequestException, ('The id "%s" is invalid--' 'it is already in use.' % id) def _setObject(self, id, object, roles=None, user=None, set_owner=1, suppress_events=False): ob = object # better name, keep original function signature v = self._checkId(id) if v is not None: id = v # If an object by the given id already exists, remove it. if self.has_key(id): self._delObject(id) if not suppress_events: notify(ObjectWillBeAddedEvent(ob, self, id)) self._setOb(id, ob) ob = self._getOb(id) if set_owner: # TODO: eventify manage_fixupOwnershipAfterAdd # This will be called for a copy/clone, or a normal _setObject. ob.manage_fixupOwnershipAfterAdd() # Try to give user the local role "Owner", but only if # no local roles have been set on the object yet. if getattr(ob, '__ac_local_roles__', _marker) is None: user = getSecurityManager().getUser() if user is not None: userid = user.getId() if userid is not None: ob.manage_setLocalRoles(userid, ['Owner']) if not suppress_events: notify(ObjectAddedEvent(ob, self, id)) notifyContainerModified(self) OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self) return id def _delObject(self, id, dp=1, suppress_events=False): ob = self._getOb(id) OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self) if not suppress_events: notify(ObjectWillBeRemovedEvent(ob, self, id)) self._delOb(id) if not suppress_events: notify(ObjectRemovedEvent(ob, self, id)) notifyContainerModified(self) # Aliases for mapping-like access. __len__ = objectCount keys = objectIds values = objectValues items = objectItems # backward compatibility hasObject = has_key security.declareProtected(access_contents_information, 'get') def get(self, name, default=None): return self._getOb(name, default) # Utility for generating unique IDs. security.declareProtected(access_contents_information, 'generateId') def generateId(self, prefix='item', suffix='', rand_ceiling=999999999): """Returns an ID not used yet by this folder. The ID is unlikely to collide with other threads and clients. The IDs are sequential to optimize access to objects that are likely to have some relation. """ tree = self._tree n = self._v_nextid attempt = 0 while 1: if n % 4000 != 0 and n <= rand_ceiling: id = '%s%d%s' % (prefix, n, suffix) if not tree.has_key(id): break n = randint(1, rand_ceiling) attempt = attempt + 1 if attempt > MAX_UNIQUEID_ATTEMPTS: # Prevent denial of service raise ExhaustedUniqueIdsError self._v_nextid = n + 1 return id def __getattr__(self, name): # Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal # to subitems, and __bobo_traverse__ hooks don't work with # restrictedTraverse() unless __getattr__() is also present. # Oh well. res = self._tree.get(name) if res is None: raise AttributeError, name return res
class FormSaveDataAdapter(FormActionAdapter): """A form action adapter that will save form input data and return it in csv- or tab-delimited format.""" schema = FormAdapterSchema.copy() + Schema(( LinesField('ExtraData', widget=MultiSelectionWidget( label=_(u'label_savedataextra_text', default='Extra Data'), description=_(u'help_savedataextra_text', default=u""" Pick any extra data you'd like saved with the form input. """), format='checkbox', ), vocabulary = 'vocabExtraDataDL', ), StringField('DownloadFormat', searchable=0, required=1, default='csv', vocabulary = 'vocabFormatDL', widget=SelectionWidget( label=_(u'label_downloadformat_text', default=u'Download Format'), ), ), BooleanField("UseColumnNames", required=False, searchable=False, widget=BooleanWidget( label = _(u'label_usecolumnnames_text', default=u"Include Column Names"), description = _(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"), ), ), ExLinesField('SavedFormInput', edit_accessor='getSavedFormInputForEdit', mutator='setSavedFormInput', searchable=0, required=0, primary=1, schemata="saved data", read_permission=DOWNLOAD_SAVED_PERMISSION, widget=TextAreaWidget( label=_(u'label_savedatainput_text', default=u"Saved Form Input"), description=_(u'help_savedatainput_text'), ), ), )) schema.moveField('execCondition', pos='bottom') meta_type = 'FormSaveDataAdapter' portal_type = 'FormSaveDataAdapter' archetype_name = 'Save Data Adapter' immediate_view = 'fg_savedata_view_p3' default_view = 'fg_savedata_view_p3' suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',) security = ClassSecurityInfo() def _migrateStorage(self): # we're going to use an LOBTree for storage. we need to # consider the possibility that self is from an # older version that uses the native Archetypes storage # or the former IOBTree (<= 1.6.0b2 ) # in the SavedFormInput field. updated = base_hasattr(self, '_inputStorage') and \ base_hasattr(self, '_inputItems') and \ base_hasattr(self, '_length') if not updated: try: saved_input = self.getSavedFormInput() except AttributeError: saved_input = [] self._inputStorage = SavedDataBTree() i = 0 self._inputItems = 0 self._length = Length() if len(saved_input): for row in saved_input: self._inputStorage[i] = row i += 1 self.SavedFormInput = [] self._inputItems = i self._length.set(i) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput') def getSavedFormInput(self): """ returns saved input as an iterable; each row is a sequence of fields. """ if base_hasattr(self, '_inputStorage'): return self._inputStorage.values() else: return self.SavedFormInput security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems') def getSavedFormInputItems(self): """ returns saved input as an iterable; each row is an (id, sequence of fields) tuple """ if base_hasattr(self, '_inputStorage'): return self._inputStorage.items() else: return enumerate(self.SavedFormInput) security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit') def getSavedFormInputForEdit(self, **kwargs): """ returns saved as CSV text """ delimiter = self.csvDelimiter() sbuf = StringIO() writer = csv.writer(sbuf, delimiter=delimiter) for row in self.getSavedFormInput(): writer.writerow( row ) res = sbuf.getvalue() sbuf.close() return res security.declareProtected(ModifyPortalContent, 'setSavedFormInput') def setSavedFormInput(self, value, **kwargs): """ expects value as csv text string, stores as list of lists """ self._migrateStorage() self._inputStorage.clear() i = 0 self._inputItems = 0 self._length.set(0) if len(value): delimiter = self.csvDelimiter() sbuf = StringIO( value ) reader = csv.reader(sbuf, delimiter=delimiter) for row in reader: if row: self._inputStorage[i] = row i += 1 self._inputItems = i self._length.set(i) sbuf.close() # logger.debug("setSavedFormInput: %s items" % self._inputItems) security.declareProtected(ModifyPortalContent, 'clearSavedFormInput') def clearSavedFormInput(self, **kwargs): """ convenience method to clear input buffer """ REQUEST = kwargs.get('request', self.REQUEST) if REQUEST.form.has_key('clearSavedFormInput'): # we're processing a request from the web; # check for CSRF plone.protect.CheckAuthenticator(REQUEST) plone.protect.PostOnly(REQUEST) self._migrateStorage() self._inputStorage.clear() self._inputItems = 0 self._length.set(0) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById') def getSavedFormInputById(self, id): """ Return the data stored for record with 'id' """ lst = [field.replace('\r','').replace('\n', r'\n') for field in self._inputStorage[id]] return lst security.declareProtected(ModifyPortalContent, 'manage_saveData') def manage_saveData(self, id, data): """ Save the data for record with 'id' """ self._migrateStorage() lst = list() for i in range(0, len(self.getColumnNames())): lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n')) self._inputStorage[id] = lst self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view') security.declareProtected(ModifyPortalContent, 'manage_deleteData') def manage_deleteData(self, id): """ Delete the data for record with 'id' """ self._migrateStorage() del self._inputStorage[id] self._inputItems -= 1 self._length.change(-1) self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view') def _addDataRow(self, value): self._migrateStorage() if isinstance(self._inputStorage, IOBTree): # 32-bit IOBTree; use a key which is more likely to conflict # but which won't overflow the key's bits id = self._inputItems self._inputItems += 1 else: # 64-bit LOBTree id = int(time.time() * 1000) while id in self._inputStorage: # avoid collisions during testing id += 1 self._inputStorage[id] = value self._length.change(1) security.declareProtected(ModifyPortalContent, 'addDataRow') def addDataRow(self, value): """ a wrapper for the _addDataRow method """ self._addDataRow(value) def onSuccess(self, fields, REQUEST=None, loopstop=False): """ saves data. """ if LP_SAVE_TO_CANONICAL and not loopstop: # LinguaPlone functionality: # check to see if we're in a translated # form folder, but not the canonical version. parent = self.aq_parent if safe_hasattr(parent, 'isTranslation') and \ parent.isTranslation() and not parent.isCanonical(): # look in the canonical version to see if there is # a matching (by id) save-data adapter. # If so, call its onSuccess method cf = parent.getCanonical() target = cf.get(self.getId()) if target is not None and target.meta_type == 'FormSaveDataAdapter': target.onSuccess(fields, REQUEST, loopstop=True) return from ZPublisher.HTTPRequest import FileUpload data = [] for f in fields: if f.isFileField(): file = REQUEST.form.get('%s_file' % f.fgField.getName()) if isinstance(file, FileUpload) and file.filename != '': file.seek(0) fdata = file.read() filename = file.filename mimetype, enc = guess_content_type(filename, fdata, None) if mimetype.find('text/') >= 0: # convert to native eols fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n') data.append( '%s:%s:%s:%s' % (filename, mimetype, enc, fdata) ) else: data.append( '%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc) ) else: data.append( 'NO UPLOAD' ) elif not f.isLabel(): val = REQUEST.form.get(f.fgField.getName(),'') if not type(val) in StringTypes: # Zope has marshalled the field into # something other than a string val = str(val) data.append(val) if self.ExtraData: for f in self.ExtraData: if f == 'dt': data.append( str(DateTime()) ) else: data.append( getattr(REQUEST, f, '') ) self._addDataRow( data ) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames') def getColumnNames(self): """Returns a list of column names""" names = [field.getName() for field in self.fgFields(displayOnly=True)] for f in self.ExtraData: names.append(f) return names security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles') def getColumnTitles(self): """Returns a list of column titles""" names = [field.widget.label for field in self.fgFields(displayOnly=True)] for f in self.ExtraData: names.append(self.vocabExtraDataDL().getValue(f, '')) return names def _cleanInputForTSV(self, value): # make data safe to store in tab-delimited format return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t') security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv') def download_tsv(self, REQUEST=None, RESPONSE=None): """Download the saved data """ filename = self.id if filename.find('.') < 0: filename = '%s.tsv' % filename header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset()) if getattr(self, 'UseColumnNames', False): res = "%s\n" % '\t'.join( self.getColumnNames() ) if isinstance(res, unicode): res = res.encode(self.getCharset()) else: res = '' for row in self.getSavedFormInput(): res = '%s%s\n' % (res, '\t'.join( [self._cleanInputForTSV(col) for col in row] )) return res security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv') def download_csv(self, REQUEST=None, RESPONSE=None): """Download the saved data """ filename = self.id if filename.find('.') < 0: filename = '%s.csv' % filename header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset()) if getattr(self, 'UseColumnNames', False): delimiter = self.csvDelimiter() res = "%s\n" % delimiter.join( self.getColumnNames() ) if isinstance(res, unicode): res = res.encode(self.getCharset()) else: res = '' return '%s%s' % (res, self.getSavedFormInputForEdit()) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download') def download(self, REQUEST=None, RESPONSE=None): """Download the saved data """ format = getattr(self, 'DownloadFormat', 'tsv') if format == 'tsv': return self.download_tsv(REQUEST, RESPONSE) else: assert format == 'csv', 'Unknown download format' return self.download_csv(REQUEST, RESPONSE) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict') def rowAsColDict(self, row, cols): """ Where row is a data sequence and cols is a column name sequence, returns a dict of colname:column. This is a convenience method used in the record view. """ colcount = len(cols) rdict = {} for i in range(0, len(row)): if i < colcount: rdict[cols[i]] = row[i] else: rdict['column-%s' % i] = row[i] return rdict security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries') def inputAsDictionaries(self): """returns saved data as an iterable of dictionaries """ cols = self.getColumnNames() for row in self.getSavedFormInput(): yield self.rowAsColDict(row, cols) # alias for old mis-naming security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries') InputAsDictionaries = inputAsDictionaries security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME') def formatMIME(self): """MIME format selected for download """ format = getattr(self, 'DownloadFormat', 'tsv') if format == 'tsv': return 'text/tab-separated-values' else: assert format == 'csv', 'Unknown download format' return 'text/comma-separated-values' security.declarePrivate('csvDelimiter') def csvDelimiter(self): """Delimiter character for CSV downloads """ fgt = getToolByName(self, 'formgen_tool') return fgt.getCSVDelimiter() security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved') def itemsSaved(self): """Download the saved data """ if base_hasattr(self, '_length'): return self._length() elif base_hasattr(self, '_inputItems'): return self._inputItems else: return len(self.SavedFormInput) def vocabExtraDataDL(self): """ returns vocabulary for extra data """ return DisplayList( ( ('dt', self.translate( msgid='vocabulary_postingdt_text', domain='ploneformgen', default='Posting Date/Time') ), ('HTTP_X_FORWARDED_FOR','HTTP_X_FORWARDED_FOR',), ('REMOTE_ADDR','REMOTE_ADDR',), ('HTTP_USER_AGENT','HTTP_USER_AGENT',), ) ) def vocabFormatDL(self): """ returns vocabulary for format """ return DisplayList( ( ('tsv', self.translate( msgid='vocabulary_tsv_text', domain='ploneformgen', default='Tab-Separated Values') ), ('csv', self.translate( msgid='vocabulary_csv_text', domain='ploneformgen', default='Comma-Separated Values') ), ) )
class HBTreeFolder2Base (Persistent): """Base for BTree-based folders. """ security = ClassSecurityInfo() manage_options=( ({'label':'Contents', 'action':'manage_main',}, ) + Folder.manage_options[1:] ) security.declareProtected(view_management_screens, 'manage_main') manage_main = DTMLFile('contents', globals()) _htree = None # OOBTree: { id -> object } _count = None # A BTrees.Length _v_nextid = 0 # The integer component of the next generated ID title = '' _tree_list = None def __init__(self, id=None): if id is not None: self.id = id self._initBTrees() def _initBTrees(self): self._htree = OOBTree() self._count = Length() self._tree_list = PersistentMapping() def _populateFromFolder(self, source): """Fill this folder with the contents of another folder. """ for name in source.objectIds(): value = source._getOb(name, None) if value is not None: self._setOb(name, aq_base(value)) security.declareProtected(view_management_screens, 'manage_fixCount') def manage_fixCount(self): """Calls self._fixCount() and reports the result as text. """ old, new = self._fixCount() path = '/'.join(self.getPhysicalPath()) if old == new: return "No count mismatch detected in HBTreeFolder2 at %s." % path else: return ("Fixed count mismatch in HBTreeFolder2 at %s. " "Count was %d; corrected to %d" % (path, old, new)) def _fixCount(self): """Checks if the value of self._count disagrees with len(self.objectIds()). If so, corrects self._count. Returns the old and new count values. If old==new, no correction was performed. """ old = self._count() new = len(self.objectIds()) if old != new: self._count.set(new) return old, new security.declareProtected(view_management_screens, 'manage_cleanup') def manage_cleanup(self): """Calls self._cleanup() and reports the result as text. """ v = self._cleanup() path = '/'.join(self.getPhysicalPath()) if v: return "No damage detected in HBTreeFolder2 at %s." % path else: return ("Fixed HBTreeFolder2 at %s. " "See the log for more details." % path) def _cleanup(self): """Cleans up errors in the BTrees. Certain ZODB bugs have caused BTrees to become slightly insane. Fortunately, there is a way to clean up damaged BTrees that always seems to work: make a new BTree containing the items() of the old one. Returns 1 if no damage was detected, or 0 if damage was detected and fixed. """ def hCheck(htree): """ Recursively check the btree """ check(htree) for key in htree.keys(): if not htree.has_key(key): raise AssertionError( "Missing value for key: %s" % repr(key)) else: ob = htree[key] if isinstance(ob, OOBTree): hCheck(ob) return 1 from BTrees.check import check path = '/'.join(self.getPhysicalPath()) try: return hCheck(self._htree) except AssertionError: LOG('HBTreeFolder2', WARNING, 'Detected damage to %s. Fixing now.' % path, error=sys.exc_info()) try: self._htree = OOBTree(self._htree) # XXX hFix needed except: LOG('HBTreeFolder2', ERROR, 'Failed to fix %s.' % path, error=sys.exc_info()) raise else: LOG('HBTreeFolder2', INFO, 'Fixed %s.' % path) return 0 def hashId(self, id): """Return a tuple of ids """ # XXX: why tolerate non-string ids ? id_list = str(id).split(H_SEPARATOR) # We use '-' as the separator by default if len(id_list) > 1: return tuple(id_list) else: return [id,] # try: # We then try int hashing # id_int = int(id) # except ValueError: # return id_list # result = [] # while id_int: # result.append(id_int % MAX_OBJECT_PER_LEVEL) # id_int = id_int / MAX_OBJECT_PER_LEVEL # result.reverse() # return tuple(result) def _getOb(self, id, default=_marker): """ Return the named object from the folder. """ htree = self._htree ob = htree id_list = self.hashId(id) for sub_id in id_list[0:-1]: if default is _marker: ob = ob[sub_id] else: ob = ob.get(sub_id, _marker) if ob is _marker: return default if default is _marker: ob = ob[id] else: ob = ob.get(id, _marker) if ob is _marker: return default return ob.__of__(self) def _setOb(self, id, object): """Store the named object in the folder. """ htree = self._htree id_list = self.hashId(id) for idx in xrange(len(id_list) - 1): sub_id = id_list[idx] if not htree.has_key(sub_id): # Create a new level htree[sub_id] = OOBTree() if isinstance(sub_id, (int, long)): tree_id = 0 for id in id_list[:idx+1]: tree_id = tree_id + id * MAX_OBJECT_PER_LEVEL else: tree_id = H_SEPARATOR.join(id_list[:idx+1]) # Index newly created level self._tree_list[tree_id] = None htree = htree[sub_id] if len(id_list) == 1 and not htree.has_key(None): self._tree_list[None] = None # set object in subtree ob_id = id_list[-1] if htree.has_key(id): raise KeyError('There is already an item named "%s".' % id) htree[id] = object self._count.change(1) def _delOb(self, id): """Remove the named object from the folder. """ htree = self._htree id_list = self.hashId(id) for sub_id in id_list[0:-1]: htree = htree[sub_id] del htree[id] self._count.change(-1) security.declareProtected(view_management_screens, 'getBatchObjectListing') def getBatchObjectListing(self, REQUEST=None): """Return a structure for a page template to show the list of objects. """ if REQUEST is None: REQUEST = {} pref_rows = int(REQUEST.get('dtpref_rows', 20)) b_start = int(REQUEST.get('b_start', 1)) b_count = int(REQUEST.get('b_count', 1000)) b_end = b_start + b_count - 1 url = self.absolute_url() + '/manage_main' count = self.objectCount() if b_end < count: next_url = url + '?b_start=%d' % (b_start + b_count) else: b_end = count next_url = '' if b_start > 1: prev_url = url + '?b_start=%d' % max(b_start - b_count, 1) else: prev_url = '' formatted = [listtext0 % pref_rows] for optID in islice(self.objectIds(), b_start - 1, b_end): optID = escape(optID) formatted.append(listtext1 % (escape(optID, quote=1), optID)) formatted.append(listtext2) return {'b_start': b_start, 'b_end': b_end, 'prev_batch_url': prev_url, 'next_batch_url': next_url, 'formatted_list': ''.join(formatted)} security.declareProtected(view_management_screens, 'manage_object_workspace') def manage_object_workspace(self, ids=(), REQUEST=None): '''Redirects to the workspace of the first object in the list.''' if ids and REQUEST is not None: REQUEST.RESPONSE.redirect( '%s/%s/manage_workspace' % ( self.absolute_url(), quote(ids[0]))) else: return self.manage_main(self, REQUEST) security.declareProtected(access_contents_information, 'tpValues') def tpValues(self): """Ensures the items don't show up in the left pane. """ return () security.declareProtected(access_contents_information, 'objectCount') def objectCount(self): """Returns the number of items in the folder.""" return self._count() security.declareProtected(access_contents_information, 'has_key') def has_key(self, id): """Indicates whether the folder has an item by ID. """ htree = self._htree id_list = self.hashId(id) for sub_id in id_list[0:-1]: if not isinstance(htree, OOBTree): return 0 if not htree.has_key(sub_id): return 0 htree = htree[sub_id] if not htree.has_key(id): return 0 return 1 # Work around for the performance regression introduced in Zope 2.12.23. # Otherwise, we use superclass' __contains__ implementation, which uses # objectIds, which is inefficient in HBTreeFolder2 to lookup a single key. __contains__ = has_key def _htree_iteritems(self, min=None): # BUG: Due to bad design of HBTreeFolder2, buckets other than the root # one must not contain both buckets & leafs. Otherwise, this method # fails. h = self._htree recurse_stack = [] try: for sub_id in min and self.hashId(min) or ('',): if recurse_stack: i.next() if type(h) is not OOBTree: break id += H_SEPARATOR + sub_id if type(h.itervalues().next()) is not OOBTree: sub_id = id else: id = sub_id i = h.iteritems(sub_id) recurse_stack.append(i) h = h[sub_id] except (KeyError, StopIteration): pass while recurse_stack: i = recurse_stack.pop() try: while 1: id, h = i.next() if type(h) is OOBTree: recurse_stack.append(i) i = h.iteritems() else: yield id, h except StopIteration: pass security.declareProtected(access_contents_information, 'treeIds') def treeIds(self, base_id=None): """ Return a list of subtree ids """ tree = self._getTree(base_id=base_id) return [k for k, v in self._htree.items() if isinstance(v, OOBTree)] def _getTree(self, base_id): """ Return the tree wich has the base_id """ htree = self._htree id_list = self.hashId(base_id) for sub_id in id_list: if not isinstance(htree, OOBTree): return None if not htree.has_key(sub_id): raise IndexError, base_id htree = htree[sub_id] return htree def _getTreeIdList(self, htree=None): """ recursively build a list of btree ids """ if htree is None: htree = self._htree btree_list = [] else: btree_list = [] for obj_id in htree.keys(): obj = htree[obj_id] if isinstance(obj, OOBTree): btree_list.extend(["%s-%s"%(obj_id, x) for x in self._getTreeIdList(htree=obj)]) btree_list.append(obj_id) return btree_list security.declareProtected(access_contents_information, 'getTreeIdList') def getTreeIdList(self, htree=None): """ Return list of all tree ids """ if self._tree_list is None or len(self._tree_list.keys()) == 0: tree_list = self._getTreeIdList(htree=htree) self._tree_list = PersistentMapping() for tree in tree_list: self._tree_list[tree] = None return sorted(self._tree_list.keys()) def _checkObjectId(self, ids): """ test id is not in btree id list """ base_id, obj_id = ids if base_id is not None: obj_id = "%s%s%s" %(base_id, H_SEPARATOR, obj_id) return not self._tree_list.has_key(obj_id) security.declareProtected(access_contents_information, 'objectValues') def objectValues(self, base_id=_marker): return HBTreeObjectValues(self, base_id) security.declareProtected(access_contents_information, 'objectIds') def objectIds(self, base_id=_marker): return HBTreeObjectIds(self, base_id) security.declareProtected(access_contents_information, 'objectItems') def objectItems(self, base_id=_marker): # Returns a list of (id, subobject) tuples of the current object. # If 'spec' is specified, returns only objects whose meta_type match # 'spec' return HBTreeObjectItems(self, base_id) # superValues() looks for the _objects attribute, but the implementation # would be inefficient, so superValues() support is disabled. _objects = () security.declareProtected(access_contents_information, 'objectIds_d') def objectIds_d(self, t=None): return dict.fromkeys(self.objectIds(t), 1) def _checkId(self, id, allow_dup=0): if not allow_dup and self.has_key(id): raise BadRequestException, ('The id "%s" is invalid--' 'it is already in use.' % id) def _setObject(self, id, object, roles=None, user=None, set_owner=1): v=self._checkId(id) if v is not None: id=v # If an object by the given id already exists, remove it. if self.has_key(id): self._delObject(id) self._setOb(id, object) object = self._getOb(id) if set_owner: object.manage_fixupOwnershipAfterAdd() # Try to give user the local role "Owner", but only if # no local roles have been set on the object yet. if hasattr(object, '__ac_local_roles__'): if object.__ac_local_roles__ is None: user=getSecurityManager().getUser() if user is not None: userid=user.getId() if userid is not None: object.manage_setLocalRoles(userid, ['Owner']) object.manage_afterAdd(object, self) return id def _delObject(self, id, dp=1): object = self._getOb(id) try: object.manage_beforeDelete(object, self) except BeforeDeleteException, ob: raise except ConflictError: raise
class HBTreeFolder2Base (Persistent): """Base for BTree-based folders. BUG: Due to wrong design, we can't store 2 objects <A> and <A>-<B> where <A> does not contain '-'. We detect conflicts at the root level using 'type(ob) is OOBTree' """ security = ClassSecurityInfo() manage_options=( ({'label':'Contents', 'action':'manage_main',}, ) + Folder.manage_options[1:] ) security.declareProtected(view_management_screens, 'manage_main') manage_main = DTMLFile('contents', globals()) _htree = None # OOBTree: { id -> object } _count = None # A BTrees.Length _v_nextid = 0 # The integer component of the next generated ID title = '' def __init__(self, id=None): if id is not None: self.id = id self._initBTrees() def _initBTrees(self): self._htree = OOBTree() self._count = Length() def _populateFromFolder(self, source): """Fill this folder with the contents of another folder. """ for name, value in source.objectItems(): self._setOb(name, aq_base(value)) security.declareProtected(view_management_screens, 'manage_fixCount') def manage_fixCount(self, dry_run=0): """Calls self._fixCount() and reports the result as text. """ old, new = self._fixCount(dry_run) path = '/'.join(self.getPhysicalPath()) if old == new: return "No count mismatch detected in HBTreeFolder2 at %s." % path else: return ("Fixed count mismatch in HBTreeFolder2 at %s. " "Count was %d; corrected to %d" % (path, old, new)) def _fixCount(self, dry_run=0): """Checks if the value of self._count disagrees with the content of the htree. If so, corrects self._count. Returns the old and new count values. If old==new, no correction was performed. """ old = self._count() new = sum(1 for x in self._htree_iteritems()) if old != new and not dry_run: self._count.set(new) return old, new def hashId(self, id): return id.split(H_SEPARATOR) def _htree_get(self, id): id_list = self.hashId(id) if len(id_list) == 1: ob = self._htree[id] if type(ob) is OOBTree: raise KeyError else: ob = self._htree[id_list.pop(0)] if type(ob) is not OOBTree: raise KeyError id_list[-1] = id for sub_id in id_list: ob = ob[sub_id] return ob def _getOb(self, id, default=_marker): """Return the named object from the folder """ try: return self._htree_get(id).__of__(self) except KeyError: if default is _marker: raise KeyError(id) return default def __getitem__(self, id): try: return self._htree_get(id).__of__(self) except KeyError: raise KeyError(id) def _setOb(self, id, object): """Store the named object in the folder. """ if type(object) is OOBTree: raise ValueError('HBTreeFolder2 can not store OOBTree objects') htree = self._htree for sub_id in self.hashId(id)[:-1]: try: htree = htree[sub_id] except KeyError: htree[sub_id] = htree = OOBTree() continue if type(htree) is not OOBTree: assert self._htree[sub_id] is htree, (htree, id) raise KeyError('There is already an item whose id is %r' % sub_id) if htree.has_key(id): raise KeyError('There is already an item named %r.' % id) htree[id] = object self._count.change(1) def _delOb(self, id): """Remove the named object from the folder. """ htree = self._htree h = [] for sub_id in self.hashId(id)[:-1]: h.append((htree, sub_id)) htree = htree.get(sub_id) if type(htree) is not OOBTree: raise KeyError(id) if type(htree[id]) is OOBTree: raise KeyError(id) del htree[id] self._count.change(-1) while h and not htree: htree, sub_id = h.pop() del htree[sub_id] security.declareProtected(view_management_screens, 'getBatchObjectListing') def getBatchObjectListing(self, REQUEST=None): """Return a structure for a page template to show the list of objects. """ if REQUEST is None: REQUEST = {} pref_rows = int(REQUEST.get('dtpref_rows', 20)) b_start = int(REQUEST.get('b_start', 1)) b_count = int(REQUEST.get('b_count', 1000)) b_end = b_start + b_count - 1 url = self.absolute_url() + '/manage_main' count = self.objectCount() if b_end < count: next_url = url + '?b_start=%d' % (b_start + b_count) else: b_end = count next_url = '' if b_start > 1: prev_url = url + '?b_start=%d' % max(b_start - b_count, 1) else: prev_url = '' formatted = [listtext0 % pref_rows] for optID in islice(self.objectIds(), b_start - 1, b_end): optID = escape(optID) formatted.append(listtext1 % (escape(optID, quote=1), optID)) formatted.append(listtext2) return {'b_start': b_start, 'b_end': b_end, 'prev_batch_url': prev_url, 'next_batch_url': next_url, 'formatted_list': ''.join(formatted)} security.declareProtected(view_management_screens, 'manage_object_workspace') def manage_object_workspace(self, ids=(), REQUEST=None): '''Redirects to the workspace of the first object in the list.''' if ids and REQUEST is not None: REQUEST.RESPONSE.redirect( '%s/%s/manage_workspace' % ( self.absolute_url(), quote(ids[0]))) else: return self.manage_main(self, REQUEST) security.declareProtected(access_contents_information, 'tpValues') def tpValues(self): """Ensures the items don't show up in the left pane. """ return () security.declareProtected(access_contents_information, 'objectCount') def objectCount(self): """Returns the number of items in the folder.""" return self._count() security.declareProtected(access_contents_information, 'has_key') def has_key(self, id): """Indicates whether the folder has an item by ID. """ try: self._htree_get(id) except KeyError: return 0 return 1 # Work around for the performance regression introduced in Zope 2.12.23. # Otherwise, we use superclass' __contains__ implementation, which uses # objectIds, which is inefficient in HBTreeFolder2 to lookup a single key. __contains__ = has_key def _htree_iteritems(self, min=None): # BUG: Due to bad design of HBTreeFolder2, buckets other than the root # one must not contain both buckets & leafs. Otherwise, this method # fails. h = self._htree recurse_stack = [] try: for sub_id in self.hashId(min) if min else ('',): if recurse_stack: i.next() if type(h) is not OOBTree: break id += H_SEPARATOR + sub_id if type(h.itervalues().next()) is not OOBTree: sub_id = id else: id = sub_id i = h.iteritems(sub_id) recurse_stack.append(i) h = h[sub_id] except (KeyError, StopIteration): pass while recurse_stack: i = recurse_stack.pop() try: while 1: id, h = i.next() if type(h) is OOBTree: recurse_stack.append(i) i = h.iteritems() else: yield id, h except StopIteration: pass security.declareProtected(access_contents_information, 'getTreeIdList') def getTreeIdList(self, htree=None): """ Return list of all tree ids """ r = [] s = [(None, self._htree.iteritems())] while s: base_id, items = s.pop() if base_id: for k, v in items: if type(v) is not OOBTree: r.append(base_id) # As an optimization, and because _htree_iteritems does not # support mixed buckets except at the root, we consider that # this one only contains leafs. break s.append((base_id + H_SEPARATOR + k, v.iteritems())) else: for k, v in items: if type(v) is not OOBTree: r.append(base_id) for k, v in items: if type(v) is OOBTree: s.append((k, v.iteritems())) break s.append((k, v.iteritems())) r.sort() return r security.declareProtected(access_contents_information, 'objectValues') def objectValues(self, base_id=_marker): return HBTreeObjectValues(self, base_id) security.declareProtected(access_contents_information, 'objectIds') def objectIds(self, base_id=_marker): return HBTreeObjectIds(self, base_id) security.declareProtected(access_contents_information, 'objectItems') def objectItems(self, base_id=_marker): # Returns a list of (id, subobject) tuples of the current object. return HBTreeObjectItems(self, base_id) # superValues() looks for the _objects attribute, but the implementation # would be inefficient, so superValues() support is disabled. _objects = () security.declareProtected(access_contents_information, 'objectIds_d') def objectIds_d(self, t=None): return dict.fromkeys(self.objectIds(t), 1) def _checkId(self, id, allow_dup=0): if not allow_dup and self.has_key(id): raise BadRequestException, ('The id "%s" is invalid--' 'it is already in use.' % id) def _setObject(self, id, object, roles=None, user=None, set_owner=1): v=self._checkId(id) if v is not None: id=v # If an object by the given id already exists, remove it. if self.has_key(id): self._delObject(id) self._setOb(id, object) object = self._getOb(id) if set_owner: object.manage_fixupOwnershipAfterAdd() # Try to give user the local role "Owner", but only if # no local roles have been set on the object yet. if hasattr(object, '__ac_local_roles__'): if object.__ac_local_roles__ is None: user=getSecurityManager().getUser() if user is not None: userid=user.getId() if userid is not None: object.manage_setLocalRoles(userid, ['Owner']) object.manage_afterAdd(object, self) return id def _delObject(self, id, dp=1): object = self._getOb(id) try: object.manage_beforeDelete(object, self) except BeforeDeleteException, ob: raise except ConflictError: raise
class FormXLSSaveDataAdapter(ATCTFolder, FormActionAdapter): """A form action adapter that will save form input data and return it in XLS format. Based on SaveDataAdapter """ implements(IFormXLSSaveDataAdapter) schema = FormAdapterSchema.copy() + atapi.Schema(( atapi.BooleanField("storeAttachments", required=False, default=True, widget=atapi.BooleanWidget( label = _(u'label_storeAttachments_text', default=u"Store attachments?"), description = _(u'help_storeAttachments_text', default=u"If checked, FileFields content will be saved inside this save adapter. " u"Link to the attachment will be part of XLS file. " u"Please note, if unchecked, attachments may be lost in the Universe."), ), ), atapi.LinesField('ExtraData', widget=atapi.MultiSelectionWidget( label=_(u'label_savedataextra_text', default='Extra Data'), description=_(u'help_savedataextra_text', default=u""" Pick any extra data you'd like saved with the form input. """), format='checkbox', ), vocabulary = 'vocabExtraDataDL', ), atapi.BooleanField("UseColumnNames", required=False, searchable=False, widget=atapi.BooleanWidget( label = _(u'label_usecolumnnames_text', default=u"Include Column Names"), description = _(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"), ), ), )) schema.moveField('execCondition', pos='bottom') security = ClassSecurityInfo() def _setupStorage(self): set_up = base_hasattr(self, '_inputStorage') and \ base_hasattr(self, '_inputItems') and \ base_hasattr(self, '_length') if not set_up: self._inputStorage = SavedDataBTree() self._inputItems = 0 self._length = Length() security.declarePrivate('clear') def clear(self): self._inputStorage.clear() self._inputItems = 0 self._length.set(0) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'listData') def listData(self): self._setupStorage() # be sure we have _inputStorage for item in self._inputStorage.values(): yield item def _addDataRow(self, value): self._setupStorage() if isinstance(self._inputStorage, IOBTree): # 32-bit IOBTree; use a key which is more likely to conflict # but which won't overflow the key's bits id = self._inputItems self._inputItems += 1 else: # 64-bit LOBTree id = int(time.time() * 1000) while id in self._inputStorage: # avoid collisions during testing id += 1 self._inputStorage[id] = value self._length.change(1) security.declareProtected(ModifyPortalContent, 'addDataRow') def addDataRow(self, value): """ a wrapper for the _addDataRow method """ self._addDataRow(value) def onSuccess(self, fields, REQUEST=None, loopstop=False): """ saves data. """ if LP_SAVE_TO_CANONICAL and not loopstop: # LinguaPlone functionality: # check to see if we're in a translated # form folder, but not the canonical version. parent = self.aq_parent if safe_hasattr(parent, 'isTranslation') and \ parent.isTranslation() and not parent.isCanonical(): # look in the canonical version to see if there is # a matching (by id) save-data adapter. # If so, call its onSuccess method cf = parent.getCanonical() target = cf.get(self.getId()) if target is not None and target.meta_type == 'FormXLSSaveDataAdapter': target.onSuccess(fields, REQUEST, loopstop=True) return from ZPublisher.HTTPRequest import FileUpload data = dict() for f in fields: name = f.fgField.getName() if f.isFileField(): file = REQUEST.form.get('%s_file' % name) if isinstance(file, FileUpload) and file.filename != '': if self.getStoreAttachments(): file.seek(0) fdata = file.read() filename = file.filename mimetype, enc = guess_content_type(filename, fdata, None) # create File object _id = get_safe_id(REQUEST, self, filename) obj = _createObjectByType('FormXLSSaveDataFile', self, _id) file.seek(0) obj.setTitle(filename) obj.setFile(file) obj.unmarkCreationFlag() obj.reindexObject() url = obj.absolute_url() data[name] = url else: data[name] = 'DISCARDED' else: data[name] = 'NO UPLOAD' elif not f.isLabel(): val = f.htmlValue(REQUEST) data[name] = val if self.ExtraData: for f in self.ExtraData: name = 'extra_' + f if f == 'dt': data[name] = str(DateTime()) else: if f in REQUEST: data[name] = REQUEST.get(f) self._addDataRow( data ) security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames') def getColumnNames(self): """Returns a list of column names""" names = [field.getName() for field in self.fgFields(displayOnly=True)] for f in self.ExtraData: names.append(f) return names security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles') def getColumnTitles(self): """Returns a list of column titles""" names = [field.widget.label for field in self.fgFields(displayOnly=True)] for f in self.ExtraData: names.append(self.vocabExtraDataDL().getValue(f, '')) return names def vocabExtraDataDL(self): """ returns vocabulary for extra data """ return atapi.DisplayList( ( ('dt', self.translate( msgid='vocabulary_postingdt_text', domain='ploneformgen', default='Posting Date/Time') ), ('HTTP_X_FORWARDED_FOR','HTTP_X_FORWARDED_FOR',), ('REMOTE_ADDR','REMOTE_ADDR',), ('HTTP_USER_AGENT','HTTP_USER_AGENT',), ) )