def _store_data(self, appstruct: dict):
"""Store data appstruct.
`comments_count` value is converted from int to :class:`Btrees.Length`,
to support ZODB conflict resultion.
"""
if self._count_field_name in appstruct: # pragma: no branch
data = getattr(self.context, self._annotation_key, {})
if self._count_field_name not in data:
counter = Length(0)
else:
counter = data[self._count_field_name]
count = appstruct[self._count_field_name]
counter.set(count)
appstruct[self._count_field_name] = counter
super()._store_data(appstruct)
def chooseName(self, name, object):
context = removeAllProxies(self.context)
next = getattr(context, '_next_id', None)
if next is None:
next = Length(0)
context._next_id = next
i = next()
name = unicode(name.strip())
orig = name
while (not name) or (name in self.context):
i += 1
name = '%s%0.2d'%(orig, i)
next.set(i+1)
self.checkName(name, object)
return name
class FormSaveDataAdapter(FormActionAdapter):
"""A form action adapter that will save form input data and
return it in csv- or tab-delimited format."""
schema = FormAdapterSchema.copy() + Schema((
LinesField('showFields',
required=0,
searchable=0,
vocabulary='allFieldDisplayList',
widget=PicklistWidget(
label=_(u'label_savefields_text', default=u"Saved Fields"),
description=_(u'help_savefields_text', default=u"""
Pick the fields whose inputs you'd like to include in
the saved data. If empty, all fields will be saved.
"""),
),
),
LinesField('ExtraData',
widget=MultiSelectionWidget(
label=_(u'label_savedataextra_text', default='Extra Data'),
description=_(u'help_savedataextra_text', default=u"""
Pick any extra data you'd like saved with the form input.
"""),
format='checkbox',
),
vocabulary='vocabExtraDataDL',
),
StringField('DownloadFormat',
searchable=0,
required=1,
default='csv',
vocabulary='vocabFormatDL',
widget=SelectionWidget(
label=_(u'label_downloadformat_text', default=u'Download Format'),
),
),
BooleanField("UseColumnNames",
required=False,
searchable=False,
widget=BooleanWidget(
label=_(u'label_usecolumnnames_text', default=u"Include Column Names"),
description=_(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"),
),
),
ExLinesField('SavedFormInput',
edit_accessor='getSavedFormInputForEdit',
mutator='setSavedFormInput',
searchable=0,
required=0,
primary=1,
schemata="saved data",
read_permission=DOWNLOAD_SAVED_PERMISSION,
widget=TextAreaWidget(
label=_(u'label_savedatainput_text', default=u"Saved Form Input"),
description=_(u'help_savedatainput_text'),
),
),
))
schema.moveField('execCondition', pos='bottom')
meta_type = 'FormSaveDataAdapter'
portal_type = 'FormSaveDataAdapter'
archetype_name = 'Save Data Adapter'
immediate_view = 'fg_savedata_view_p3'
default_view = 'fg_savedata_view_p3'
suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',)
security = ClassSecurityInfo()
def _migrateStorage(self):
# we're going to use an LOBTree for storage. we need to
# consider the possibility that self is from an
# older version that uses the native Archetypes storage
# or the former IOBTree (<= 1.6.0b2 )
# in the SavedFormInput field.
updated = base_hasattr(self, '_inputStorage') and \
base_hasattr(self, '_inputItems') and \
base_hasattr(self, '_length')
if not updated:
try:
saved_input = self.getSavedFormInput()
except AttributeError:
saved_input = []
self._inputStorage = SavedDataBTree()
i = 0
self._inputItems = 0
self._length = Length()
if len(saved_input):
for row in saved_input:
self._inputStorage[i] = row
i += 1
self.SavedFormInput = []
self._inputItems = i
self._length.set(i)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput')
def getSavedFormInput(self):
""" returns saved input as an iterable;
each row is a sequence of fields.
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.values()
else:
return self.SavedFormInput
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems')
def getSavedFormInputItems(self):
""" returns saved input as an iterable;
each row is an (id, sequence of fields) tuple
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.items()
else:
return enumerate(self.SavedFormInput)
security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit')
def getSavedFormInputForEdit(self, **kwargs):
""" returns saved as CSV text """
delimiter = self.csvDelimiter()
sbuf = StringIO()
writer = csv.writer(sbuf, delimiter=delimiter)
for row in self.getSavedFormInput():
writer.writerow(row)
res = sbuf.getvalue()
sbuf.close()
return res
security.declareProtected(ModifyPortalContent, 'setSavedFormInput')
def setSavedFormInput(self, value, **kwargs):
""" expects value as csv text string, stores as list of lists """
self._migrateStorage()
self._inputStorage.clear()
i = 0
self._inputItems = 0
self._length.set(0)
if len(value):
delimiter = self.csvDelimiter()
sbuf = StringIO(value)
reader = csv.reader(sbuf, delimiter=delimiter)
for row in reader:
if row:
self._inputStorage[i] = row
i += 1
self._inputItems = i
self._length.set(i)
sbuf.close()
# logger.debug("setSavedFormInput: %s items" % self._inputItems)
def _clearSavedFormInput(self):
# convenience method to clear input buffer
self._migrateStorage()
self._inputStorage.clear()
self._inputItems = 0
self._length.set(0)
security.declareProtected(ModifyPortalContent, 'clearSavedFormInput')
def clearSavedFormInput(self, **kwargs):
""" clear input buffer TTW """
plone.protect.CheckAuthenticator(self.REQUEST)
plone.protect.PostOnly(self.REQUEST)
self._clearSavedFormInput()
self.REQUEST.response.redirect(self.absolute_url())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById')
def getSavedFormInputById(self, id):
""" Return the data stored for record with 'id' """
lst = [field.replace('\r', '').replace('\n', r'\n') for field in self._inputStorage[id]]
return lst
security.declareProtected(ModifyPortalContent, 'manage_saveData')
def manage_saveData(self, id, data):
""" Save the data for record with 'id' """
plone.protect.CheckAuthenticator(self.REQUEST)
plone.protect.PostOnly(self.REQUEST)
self._migrateStorage()
lst = list()
for i in range(0, len(self.getColumnNames())):
lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n'))
self._inputStorage[id] = lst
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
security.declareProtected(ModifyPortalContent, 'manage_deleteData')
def manage_deleteData(self, id):
""" Delete the data for record with 'id' """
self._migrateStorage()
del self._inputStorage[id]
self._inputItems -= 1
self._length.change(-1)
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
def _addDataRow(self, value):
self._migrateStorage()
if isinstance(self._inputStorage, IOBTree):
# 32-bit IOBTree; use a key which is more likely to conflict
# but which won't overflow the key's bits
id = self._inputItems
self._inputItems += 1
else:
# 64-bit LOBTree
id = int(time.time() * 1000)
while id in self._inputStorage: # avoid collisions during testing
id += 1
self._inputStorage[id] = value
self._length.change(1)
security.declareProtected(ModifyPortalContent, 'addDataRow')
def addDataRow(self, value):
# """ a wrapper for the _addDataRow method """
self._addDataRow(value)
security.declarePrivate('onSuccess')
def onSuccess(self, fields, REQUEST=None, loopstop=False):
# """
# saves data.
# """
if LP_SAVE_TO_CANONICAL and not loopstop:
# LinguaPlone functionality:
# check to see if we're in a translated
# form folder, but not the canonical version.
parent = self.aq_parent
if safe_hasattr(parent, 'isTranslation') and \
parent.isTranslation() and not parent.isCanonical():
# look in the canonical version to see if there is
# a matching (by id) save-data adapter.
# If so, call its onSuccess method
cf = parent.getCanonical()
target = cf.get(self.getId())
if target is not None and target.meta_type == 'FormSaveDataAdapter':
target.onSuccess(fields, REQUEST, loopstop=True)
return
from ZPublisher.HTTPRequest import FileUpload
data = []
for f in fields:
showFields = getattr(self, 'showFields', [])
if showFields and f.id not in showFields:
continue
if f.isFileField():
file = REQUEST.form.get('%s_file' % f.fgField.getName())
if isinstance(file, FileUpload) and file.filename != '':
file.seek(0)
fdata = file.read()
filename = file.filename
mimetype, enc = guess_content_type(filename, fdata, None)
if mimetype.find('text/') >= 0:
# convert to native eols
fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n')
data.append('%s:%s:%s:%s' % (filename, mimetype, enc, fdata))
else:
data.append('%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc))
else:
data.append('NO UPLOAD')
elif not f.isLabel():
val = REQUEST.form.get(f.fgField.getName(), '')
if not type(val) in StringTypes:
# Zope has marshalled the field into
# something other than a string
val = str(val)
data.append(val)
if self.ExtraData:
for f in self.ExtraData:
if f == 'dt':
data.append(str(DateTime()))
else:
data.append(getattr(REQUEST, f, ''))
self._addDataRow(data)
security.declareProtected(ModifyPortalContent, 'allFieldDisplayList')
def allFieldDisplayList(self):
# """ returns a DisplayList of all fields """
return self.fgFieldsDisplayList()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames')
def getColumnNames(self, excludeServerSide=True):
# """Returns a list of column names"""
showFields = getattr(self, 'showFields', [])
names = [field.getName() for field
in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide)
if not showFields or field.getName() in showFields]
for f in self.ExtraData:
names.append(f)
return names
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles')
def getColumnTitles(self, excludeServerSide=True):
# """Returns a list of column titles"""
names = [field.widget.label for field
in self.fgFields(displayOnly=True, excludeServerSide=excludeServerSide)]
for f in self.ExtraData:
names.append(self.vocabExtraDataDL().getValue(f, ''))
return names
def _cleanInputForTSV(self, value):
# make data safe to store in tab-delimited format
return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t')
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv')
def download_tsv(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.tsv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
res = "%s\n" % '\t'.join(self.getColumnNames(excludeServerSide=False))
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
for row in self.getSavedFormInput():
res = '%s%s\n' % (res, '\t'.join([self._cleanInputForTSV(col) for col in row]))
return res
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv')
def download_csv(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.csv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
delimiter = self.csvDelimiter()
res = "%s\n" % delimiter.join(self.getColumnNames(excludeServerSide=False))
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
return '%s%s' % (res, self.getSavedFormInputForEdit())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_xls')
def download_xls(self, REQUEST=None, RESPONSE=None):
# """Download the saved data
# """
filename = self.id
if filename.find('.') < 0:
filename = '%s.xls' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'application/vnd.ms-excel')
xldoc = xlwt.Workbook(encoding=self.getCharset())
sheet = xldoc.add_sheet(self.Title())
row_num = 0
if getattr(self, 'UseColumnNames', False):
col_names = self.getColumnNames(excludeServerSide=False)
for idx, label in enumerate(col_names):
sheet.write(0, idx, label.encode(self.getCharset()))
row_num += 1
for row in self.getSavedFormInput():
for col_num, col in enumerate(row):
if type(col) is unicode:
col = col.encode(self.getCharset())
if urlparse(col).scheme in ('http', 'https'):
col = xlwt.Formula('HYPERLINK("%(url)s")' % dict(url=col))
else:
for format in (int, float):
try:
col = format(col)
break
except ValueError:
pass
sheet.write(row_num, col_num, col)
row_num += 1
string_buffer = StringIO()
xldoc.save(string_buffer)
return string_buffer.getvalue()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download')
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
if format == 'xls':
assert has_xls, 'xls download not available'
return self.download_xls(REQUEST, RESPONSE)
else:
assert format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict')
def rowAsColDict(self, row, cols):
# """ Where row is a data sequence and cols is a column name sequence,
# returns a dict of colname:column. This is a convenience method
# used in the record view.
# """
colcount = len(cols)
rdict = {}
for i in range(0, len(row)):
if i < colcount:
rdict[cols[i]] = row[i]
else:
rdict['column-%s' % i] = row[i]
return rdict
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries')
def inputAsDictionaries(self):
# """returns saved data as an iterable of dictionaries
# """
cols = self.getColumnNames()
for row in self.getSavedFormInput():
yield self.rowAsColDict(row, cols)
# alias for old mis-naming
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries')
InputAsDictionaries = inputAsDictionaries
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME')
def formatMIME(self):
# """MIME format selected for download
# """
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return 'text/tab-separated-values'
if format == 'xls':
return 'application/vnd.ms-excel'
else:
assert format == 'csv', 'Unknown download format'
return 'text/comma-separated-values'
security.declarePrivate('csvDelimiter')
def csvDelimiter(self):
# """Delimiter character for CSV downloads
# """
fgt = getToolByName(self, 'formgen_tool')
return fgt.getCSVDelimiter()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved')
def itemsSaved(self):
# """Download the saved data
# """
if base_hasattr(self, '_length'):
return self._length()
elif base_hasattr(self, '_inputItems'):
return self._inputItems
else:
return len(self.SavedFormInput)
def vocabExtraDataDL(self):
# """ returns vocabulary for extra data """
return DisplayList((
('dt',
self.translate(msgid='vocabulary_postingdt_text',
domain='ploneformgen',
default='Posting Date/Time')
),
('HTTP_X_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR',),
('REMOTE_ADDR', 'REMOTE_ADDR',),
('HTTP_USER_AGENT', 'HTTP_USER_AGENT',),
))
def vocabFormatDL(self):
# """ returns vocabulary for format """
formats = [
('tsv',
self.translate(msgid='vocabulary_tsv_text',
domain='ploneformgen',
default='Tab-Separated Values')
),
('csv',
self.translate(msgid='vocabulary_csv_text',
domain='ploneformgen',
default='Comma-Separated Values')
),
]
if has_xls:
formats.append(
('xls',
self.translate(msgid='vocabulary_xls_doc',
domain='ploneformgen',
default='Excel document')
),
)
return DisplayList(formats)
class CachingCatalog(Catalog):
implements(ICatalog)
os = os # for unit tests
generation = None # b/c
def __init__(self):
super(CachingCatalog, self).__init__()
self.generation = Length(0)
def clear(self):
self.invalidate()
super(CachingCatalog, self).clear()
def index_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).index_doc(*arg, **kw)
def unindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).unindex_doc(*arg, **kw)
def reindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).reindex_doc(*arg, **kw)
def __setitem__(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).__setitem__(*arg, **kw)
@MetricMod('CS.%s')
@metricmethod
def search(self, *arg, **kw):
use_cache = True
if 'use_cache' in kw:
use_cache = kw.pop('use_cache')
if 'NO_CATALOG_CACHE' in self.os.environ:
use_cache = False
if 'tags' in kw:
# The tags index changes without invalidating the catalog,
# so don't cache any query involving the tags index.
use_cache = False
if not use_cache:
return self._search(*arg, **kw)
cache = queryUtility(ICatalogSearchCache)
if cache is None:
return self._search(*arg, **kw)
key = cPickle.dumps((arg, kw))
generation = self.generation
if generation is None:
generation = Length(0)
genval = generation.value
if (genval == 0) or (genval > cache.generation):
# an update in another process requires that the local cache be
# invalidated
cache.clear()
cache.generation = genval
if cache.get(key) is None:
start = time.time()
num, docids = self._search(*arg, **kw)
# We don't cache large result sets because the time it takes to
# unroll the result set turns out to be far more time than it
# takes to run the search. In a particular instance using OSI's
# catalog a search that took 0.015s but returned nearly 35,295
# results took over 50s to unroll the result set for caching,
# significantly slowing search performance.
if num > LARGE_RESULT_SET:
return num, docids
# we need to unroll here; a btree-based structure may have
# a reference to its connection
docids = list(docids)
cache[key] = (num, docids)
return cache.get(key)
@metricmethod
def _search(self, *arg, **kw):
start = time.time()
res = super(CachingCatalog, self).search(*arg, **kw)
duration = time.time() - start
notify(CatalogQueryEvent(self, kw, duration, res))
return res
def invalidate(self):
# Increment the generation; this tells *another process* that
# its catalog cache needs to be cleared
generation = self.generation
if generation is None:
generation = self.generation = Length(0)
if generation.value >= sys.maxint:
# don't keep growing the generation integer; wrap at sys.maxint
self.generation.set(0)
else:
self.generation.change(1)
# Clear the cache for *this process*
cache = queryUtility(ICatalogSearchCache)
if cache is not None:
cache.clear()
cache.generation = self.generation.value
class BTreeFolder2Base (Persistent):
"""Base for BTree-based folders.
"""
security = ClassSecurityInfo()
manage_options=(
({'label':'Contents', 'action':'manage_main',},
) + Folder.manage_options[1:]
)
security.declareProtected(view_management_screens,
'manage_main')
manage_main = DTMLFile('contents', globals())
_tree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
_mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } }
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._tree = OOBTree()
self._count = Length()
self._mt_index = OOBTree()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name in source.objectIds():
value = source._getOb(name, None)
if value is not None:
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount()
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in BTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in BTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self):
"""Checks if the value of self._count disagrees with
len(self.objectIds()). If so, corrects self._count. Returns the
old and new count values. If old==new, no correction was
performed.
"""
old = self._count()
new = len(self.objectIds())
if old != new:
self._count.set(new)
return old, new
security.declareProtected(view_management_screens, 'manage_cleanup')
def manage_cleanup(self):
"""Calls self._cleanup() and reports the result as text.
"""
v = self._cleanup()
path = '/'.join(self.getPhysicalPath())
if v:
return "No damage detected in BTreeFolder2 at %s." % path
else:
return ("Fixed BTreeFolder2 at %s. "
"See the log for more details." % path)
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
check(self._tree)
for key in self._tree.keys():
if not self._tree.has_key(key):
raise AssertionError(
"Missing value for key: %s" % repr(key))
check(self._mt_index)
for key, value in self._mt_index.items():
if (not self._mt_index.has_key(key)
or self._mt_index[key] is not value):
raise AssertionError(
"Missing or incorrect meta_type index: %s"
% repr(key))
check(value)
for k in value.keys():
if not value.has_key(k):
raise AssertionError(
"Missing values for meta_type index: %s"
% repr(key))
return 1
except AssertionError:
LOG.warn( 'Detected damage to %s. Fixing now.' % path,
exc_info=True)
try:
self._tree = OOBTree(self._tree)
mt_index = OOBTree()
for key, value in self._mt_index.items():
mt_index[key] = OIBTree(value)
self._mt_index = mt_index
except:
LOG.error('Failed to fix %s.' % path,
exc_info=True)
raise
else:
LOG.info('Fixed %s.' % path)
return 0
def _getOb(self, id, default=_marker):
"""Return the named object from the folder.
"""
tree = self._tree
if default is _marker:
ob = tree[id]
return ob.__of__(self)
else:
ob = tree.get(id, _marker)
if ob is _marker:
return default
else:
return ob.__of__(self)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
tree = self._tree
if tree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
# Update the meta type index.
mti = self._mt_index
meta_type = getattr(object, 'meta_type', None)
if meta_type is not None:
ids = mti.get(meta_type, None)
if ids is None:
ids = OIBTree()
mti[meta_type] = ids
ids[id] = 1
def _delOb(self, id):
"""Remove the named object from the folder.
"""
tree = self._tree
meta_type = getattr(tree[id], 'meta_type', None)
del tree[id]
self._count.change(-1)
# Update the meta type index.
if meta_type is not None:
mti = self._mt_index
ids = mti.get(meta_type, None)
if ids is not None and ids.has_key(id):
del ids[id]
if not ids:
# Removed the last object of this meta_type.
# Prune the index.
del mti[meta_type]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
idlist = self.objectIds() # Pre-sorted.
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = []
formatted.append(listtext0 % pref_rows)
for i in range(b_start - 1, b_end):
optID = escape(idlist[i])
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect(
'%s/%s/manage_workspace' % (
self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information,
'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information,
'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
return self._tree.has_key(id)
security.declareProtected(access_contents_information,
'objectIds')
def objectIds(self, spec=None):
# Returns a list of subobject ids of the current object.
# If 'spec' is specified, returns objects whose meta_type
# matches 'spec'.
if spec is not None:
if isinstance(spec, StringType):
spec = [spec]
mti = self._mt_index
set = None
for meta_type in spec:
ids = mti.get(meta_type, None)
if ids is not None:
set = union(set, ids)
if set is None:
return ()
else:
return set.keys()
else:
return self._tree.keys()
security.declareProtected(access_contents_information,
'objectValues')
def objectValues(self, spec=None):
# Returns a list of actual subobjects of the current object.
# If 'spec' is specified, returns only objects whose meta_type
# match 'spec'.
return LazyMap(self._getOb, self.objectIds(spec))
security.declareProtected(access_contents_information,
'objectItems')
def objectItems(self, spec=None):
# Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match
# 'spec'
return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)),
self.objectIds(spec))
security.declareProtected(access_contents_information,
'objectMap')
def objectMap(self):
# Returns a tuple of mappings containing subobject meta-data.
return LazyMap(lambda (k, v):
{'id': k, 'meta_type': getattr(v, 'meta_type', None)},
self._tree.items(), self._count())
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information,
'objectIds_d')
def objectIds_d(self, t=None):
ids = self.objectIds(t)
res = {}
for id in ids:
res[id] = 1
return res
security.declareProtected(access_contents_information,
'objectMap_d')
def objectMap_d(self, t=None):
return self.objectMap()
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self, id, object, roles=None, user=None, set_owner=1,
suppress_events=False):
ob = object # better name, keep original function signature
v = self._checkId(id)
if v is not None:
id = v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
if not suppress_events:
notify(ObjectWillBeAddedEvent(ob, self, id))
self._setOb(id, ob)
ob = self._getOb(id)
if set_owner:
# TODO: eventify manage_fixupOwnershipAfterAdd
# This will be called for a copy/clone, or a normal _setObject.
ob.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if getattr(ob, '__ac_local_roles__', _marker) is None:
user = getSecurityManager().getUser()
if user is not None:
userid = user.getId()
if userid is not None:
ob.manage_setLocalRoles(userid, ['Owner'])
if not suppress_events:
notify(ObjectAddedEvent(ob, self, id))
notifyContainerModified(self)
OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self)
return id
def _delObject(self, id, dp=1, suppress_events=False):
ob = self._getOb(id)
OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self)
if not suppress_events:
notify(ObjectWillBeRemovedEvent(ob, self, id))
self._delOb(id)
if not suppress_events:
notify(ObjectRemovedEvent(ob, self, id))
notifyContainerModified(self)
# Aliases for mapping-like access.
__len__ = objectCount
keys = objectIds
values = objectValues
items = objectItems
# backward compatibility
hasObject = has_key
security.declareProtected(access_contents_information, 'get')
def get(self, name, default=None):
return self._getOb(name, default)
# Utility for generating unique IDs.
security.declareProtected(access_contents_information, 'generateId')
def generateId(self, prefix='item', suffix='', rand_ceiling=999999999):
"""Returns an ID not used yet by this folder.
The ID is unlikely to collide with other threads and clients.
The IDs are sequential to optimize access to objects
that are likely to have some relation.
"""
tree = self._tree
n = self._v_nextid
attempt = 0
while 1:
if n % 4000 != 0 and n <= rand_ceiling:
id = '%s%d%s' % (prefix, n, suffix)
if not tree.has_key(id):
break
n = randint(1, rand_ceiling)
attempt = attempt + 1
if attempt > MAX_UNIQUEID_ATTEMPTS:
# Prevent denial of service
raise ExhaustedUniqueIdsError
self._v_nextid = n + 1
return id
def __getattr__(self, name):
# Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal
# to subitems, and __bobo_traverse__ hooks don't work with
# restrictedTraverse() unless __getattr__() is also present.
# Oh well.
res = self._tree.get(name)
if res is None:
raise AttributeError, name
return res
class CachingCatalog(Catalog):
implements(ICatalog)
os = os # for unit tests
generation = None # b/c
def __init__(self):
super(CachingCatalog, self).__init__()
self.generation = Length(0)
def clear(self):
self.invalidate()
super(CachingCatalog, self).clear()
def index_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).index_doc(*arg, **kw)
def unindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).unindex_doc(*arg, **kw)
def reindex_doc(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).reindex_doc(*arg, **kw)
def __setitem__(self, *arg, **kw):
self.invalidate()
super(CachingCatalog, self).__setitem__(*arg, **kw)
def search(self, *arg, **kw):
use_cache = True
if 'use_cache' in kw:
use_cache = kw.pop('use_cache')
if 'NO_CATALOG_CACHE' in self.os.environ:
use_cache = False
if 'tags' in kw:
# The tags index changes without invalidating the catalog,
# so don't cache any query involving the tags index.
use_cache = False
if not use_cache:
return self._search(*arg, **kw)
cache = queryUtility(ICatalogSearchCache)
if cache is None:
return self._search(*arg, **kw)
key = cPickle.dumps((arg, kw))
generation = self.generation
if generation is None:
generation = Length(0)
genval = generation.value
if (genval == 0) or (genval > cache.generation):
# an update in another process requires that the local cache be
# invalidated
cache.clear()
cache.generation = genval
if cache.get(key) is None:
num, docids = self._search(*arg, **kw)
# We don't cache large result sets because the time it takes to
# unroll the result set turns out to be far more time than it
# takes to run the search. In a particular instance using OSI's
# catalog a search that took 0.015s but returned nearly 35,295
# results took over 50s to unroll the result set for caching,
# significantly slowing search performance.
if num > LARGE_RESULT_SET:
return num, docids
# we need to unroll here; a btree-based structure may have
# a reference to its connection
docids = list(docids)
cache[key] = (num, docids)
return cache.get(key)
def _search(self, *arg, **kw):
start = time.time()
res = super(CachingCatalog, self).search(*arg, **kw)
duration = time.time() - start
notify(CatalogQueryEvent(self, kw, duration, res))
return res
def invalidate(self):
# Increment the generation; this tells *another process* that
# its catalog cache needs to be cleared
generation = self.generation
if generation is None:
generation = self.generation = Length(0)
if generation.value >= sys.maxint:
# don't keep growing the generation integer; wrap at sys.maxint
self.generation.set(0)
else:
self.generation.change(1)
# Clear the cache for *this process*
cache = queryUtility(ICatalogSearchCache)
if cache is not None:
cache.clear()
cache.generation = self.generation.value
class BTreeFolder2Base(Persistent):
"""Base for BTree-based folders.
"""
security = ClassSecurityInfo()
manage_options = (({
'label': 'Contents',
'action': 'manage_main',
}, ) + Folder.manage_options[1:])
security.declareProtected(view_management_screens, 'manage_main')
manage_main = DTMLFile('contents', globals())
_tree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
_mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } }
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._tree = OOBTree()
self._count = Length()
self._mt_index = OOBTree()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name in source.objectIds():
value = source._getOb(name, None)
if value is not None:
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount()
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in BTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in BTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self):
"""Checks if the value of self._count disagrees with
len(self.objectIds()). If so, corrects self._count. Returns the
old and new count values. If old==new, no correction was
performed.
"""
old = self._count()
new = len(self.objectIds())
if old != new:
self._count.set(new)
return old, new
security.declareProtected(view_management_screens, 'manage_cleanup')
def manage_cleanup(self):
"""Calls self._cleanup() and reports the result as text.
"""
v = self._cleanup()
path = '/'.join(self.getPhysicalPath())
if v:
return "No damage detected in BTreeFolder2 at %s." % path
else:
return ("Fixed BTreeFolder2 at %s. "
"See the log for more details." % path)
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
check(self._tree)
for key in self._tree.keys():
if not self._tree.has_key(key):
raise AssertionError("Missing value for key: %s" %
repr(key))
check(self._mt_index)
for key, value in self._mt_index.items():
if (not self._mt_index.has_key(key)
or self._mt_index[key] is not value):
raise AssertionError(
"Missing or incorrect meta_type index: %s" % repr(key))
check(value)
for k in value.keys():
if not value.has_key(k):
raise AssertionError(
"Missing values for meta_type index: %s" %
repr(key))
return 1
except AssertionError:
LOG.warn('Detected damage to %s. Fixing now.' % path,
exc_info=sys.exc_info())
try:
self._tree = OOBTree(self._tree)
mt_index = OOBTree()
for key, value in self._mt_index.items():
mt_index[key] = OIBTree(value)
self._mt_index = mt_index
except:
LOG.error('Failed to fix %s.' % path, exc_info=sys.exc_info())
raise
else:
LOG.info('Fixed %s.' % path)
return 0
def _getOb(self, id, default=_marker):
"""Return the named object from the folder.
"""
tree = self._tree
if default is _marker:
ob = tree[id]
return ob.__of__(self)
else:
ob = tree.get(id, _marker)
if ob is _marker:
return default
else:
return ob.__of__(self)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
tree = self._tree
if tree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
# Update the meta type index.
mti = self._mt_index
meta_type = getattr(object, 'meta_type', None)
if meta_type is not None:
ids = mti.get(meta_type, None)
if ids is None:
ids = OIBTree()
mti[meta_type] = ids
ids[id] = 1
def _delOb(self, id):
"""Remove the named object from the folder.
"""
tree = self._tree
meta_type = getattr(tree[id], 'meta_type', None)
del tree[id]
self._count.change(-1)
# Update the meta type index.
if meta_type is not None:
mti = self._mt_index
ids = mti.get(meta_type, None)
if ids is not None and ids.has_key(id):
del ids[id]
if not ids:
# Removed the last object of this meta_type.
# Prune the index.
del mti[meta_type]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
idlist = self.objectIds() # Pre-sorted.
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = []
formatted.append(listtext0 % pref_rows)
for i in range(b_start - 1, b_end):
optID = escape(idlist[i])
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {
'b_start': b_start,
'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)
}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect('%s/%s/manage_workspace' %
(self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information, 'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information, 'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
return self._tree.has_key(id)
security.declareProtected(access_contents_information, 'objectIds')
def objectIds(self, spec=None):
# Returns a list of subobject ids of the current object.
# If 'spec' is specified, returns objects whose meta_type
# matches 'spec'.
mti = self._mt_index
if spec is None:
spec = mti.keys() #all meta types
if isinstance(spec, StringType):
spec = [spec]
set = None
for meta_type in spec:
ids = mti.get(meta_type, None)
if ids is not None:
set = union(set, ids)
if set is None:
return ()
else:
return set.keys()
security.declareProtected(access_contents_information, 'objectValues')
def objectValues(self, spec=None):
# Returns a list of actual subobjects of the current object.
# If 'spec' is specified, returns only objects whose meta_type
# match 'spec'.
return LazyMap(self._getOb, self.objectIds(spec))
security.declareProtected(access_contents_information, 'objectItems')
def objectItems(self, spec=None):
# Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match
# 'spec'
return LazyMap(lambda id, _getOb=self._getOb: (id, _getOb(id)),
self.objectIds(spec))
security.declareProtected(access_contents_information, 'objectMap')
def objectMap(self):
# Returns a tuple of mappings containing subobject meta-data.
return LazyMap(
lambda (k, v): {
'id': k,
'meta_type': getattr(v, 'meta_type', None)
}, self._tree.items(), self._count())
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information, 'objectIds_d')
def objectIds_d(self, t=None):
ids = self.objectIds(t)
res = {}
for id in ids:
res[id] = 1
return res
security.declareProtected(access_contents_information, 'objectMap_d')
def objectMap_d(self, t=None):
return self.objectMap()
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self,
id,
object,
roles=None,
user=None,
set_owner=1,
suppress_events=False):
ob = object # better name, keep original function signature
v = self._checkId(id)
if v is not None:
id = v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
if not suppress_events:
notify(ObjectWillBeAddedEvent(ob, self, id))
self._setOb(id, ob)
ob = self._getOb(id)
if set_owner:
# TODO: eventify manage_fixupOwnershipAfterAdd
# This will be called for a copy/clone, or a normal _setObject.
ob.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if getattr(ob, '__ac_local_roles__', _marker) is None:
user = getSecurityManager().getUser()
if user is not None:
userid = user.getId()
if userid is not None:
ob.manage_setLocalRoles(userid, ['Owner'])
if not suppress_events:
notify(ObjectAddedEvent(ob, self, id))
notifyContainerModified(self)
OFS.subscribers.compatibilityCall('manage_afterAdd', ob, ob, self)
return id
def _delObject(self, id, dp=1, suppress_events=False):
ob = self._getOb(id)
OFS.subscribers.compatibilityCall('manage_beforeDelete', ob, ob, self)
if not suppress_events:
notify(ObjectWillBeRemovedEvent(ob, self, id))
self._delOb(id)
if not suppress_events:
notify(ObjectRemovedEvent(ob, self, id))
notifyContainerModified(self)
# Aliases for mapping-like access.
__len__ = objectCount
keys = objectIds
values = objectValues
items = objectItems
# backward compatibility
hasObject = has_key
security.declareProtected(access_contents_information, 'get')
def get(self, name, default=None):
return self._getOb(name, default)
# Utility for generating unique IDs.
security.declareProtected(access_contents_information, 'generateId')
def generateId(self, prefix='item', suffix='', rand_ceiling=999999999):
"""Returns an ID not used yet by this folder.
The ID is unlikely to collide with other threads and clients.
The IDs are sequential to optimize access to objects
that are likely to have some relation.
"""
tree = self._tree
n = self._v_nextid
attempt = 0
while 1:
if n % 4000 != 0 and n <= rand_ceiling:
id = '%s%d%s' % (prefix, n, suffix)
if not tree.has_key(id):
break
n = randint(1, rand_ceiling)
attempt = attempt + 1
if attempt > MAX_UNIQUEID_ATTEMPTS:
# Prevent denial of service
raise ExhaustedUniqueIdsError
self._v_nextid = n + 1
return id
def __getattr__(self, name):
# Boo hoo hoo! Zope 2 prefers implicit acquisition over traversal
# to subitems, and __bobo_traverse__ hooks don't work with
# restrictedTraverse() unless __getattr__() is also present.
# Oh well.
res = self._tree.get(name)
if res is None:
raise AttributeError, name
return res
class FormSaveDataAdapter(FormActionAdapter):
"""A form action adapter that will save form input data and
return it in csv- or tab-delimited format."""
schema = FormAdapterSchema.copy() + Schema((
LinesField('ExtraData',
widget=MultiSelectionWidget(
label=_(u'label_savedataextra_text', default='Extra Data'),
description=_(u'help_savedataextra_text', default=u"""
Pick any extra data you'd like saved with the form input.
"""),
format='checkbox',
),
vocabulary = 'vocabExtraDataDL',
),
StringField('DownloadFormat',
searchable=0,
required=1,
default='csv',
vocabulary = 'vocabFormatDL',
widget=SelectionWidget(
label=_(u'label_downloadformat_text', default=u'Download Format'),
),
),
BooleanField("UseColumnNames",
required=False,
searchable=False,
widget=BooleanWidget(
label = _(u'label_usecolumnnames_text', default=u"Include Column Names"),
description = _(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"),
),
),
ExLinesField('SavedFormInput',
edit_accessor='getSavedFormInputForEdit',
mutator='setSavedFormInput',
searchable=0,
required=0,
primary=1,
schemata="saved data",
read_permission=DOWNLOAD_SAVED_PERMISSION,
widget=TextAreaWidget(
label=_(u'label_savedatainput_text', default=u"Saved Form Input"),
description=_(u'help_savedatainput_text'),
),
),
))
schema.moveField('execCondition', pos='bottom')
meta_type = 'FormSaveDataAdapter'
portal_type = 'FormSaveDataAdapter'
archetype_name = 'Save Data Adapter'
immediate_view = 'fg_savedata_view_p3'
default_view = 'fg_savedata_view_p3'
suppl_views = ('fg_savedata_tabview_p3', 'fg_savedata_recview_p3',)
security = ClassSecurityInfo()
def _migrateStorage(self):
# we're going to use an LOBTree for storage. we need to
# consider the possibility that self is from an
# older version that uses the native Archetypes storage
# or the former IOBTree (<= 1.6.0b2 )
# in the SavedFormInput field.
updated = base_hasattr(self, '_inputStorage') and \
base_hasattr(self, '_inputItems') and \
base_hasattr(self, '_length')
if not updated:
try:
saved_input = self.getSavedFormInput()
except AttributeError:
saved_input = []
self._inputStorage = SavedDataBTree()
i = 0
self._inputItems = 0
self._length = Length()
if len(saved_input):
for row in saved_input:
self._inputStorage[i] = row
i += 1
self.SavedFormInput = []
self._inputItems = i
self._length.set(i)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInput')
def getSavedFormInput(self):
""" returns saved input as an iterable;
each row is a sequence of fields.
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.values()
else:
return self.SavedFormInput
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputItems')
def getSavedFormInputItems(self):
""" returns saved input as an iterable;
each row is an (id, sequence of fields) tuple
"""
if base_hasattr(self, '_inputStorage'):
return self._inputStorage.items()
else:
return enumerate(self.SavedFormInput)
security.declareProtected(ModifyPortalContent, 'getSavedFormInputForEdit')
def getSavedFormInputForEdit(self, **kwargs):
""" returns saved as CSV text """
delimiter = self.csvDelimiter()
sbuf = StringIO()
writer = csv.writer(sbuf, delimiter=delimiter)
for row in self.getSavedFormInput():
writer.writerow( row )
res = sbuf.getvalue()
sbuf.close()
return res
security.declareProtected(ModifyPortalContent, 'setSavedFormInput')
def setSavedFormInput(self, value, **kwargs):
""" expects value as csv text string, stores as list of lists """
self._migrateStorage()
self._inputStorage.clear()
i = 0
self._inputItems = 0
self._length.set(0)
if len(value):
delimiter = self.csvDelimiter()
sbuf = StringIO( value )
reader = csv.reader(sbuf, delimiter=delimiter)
for row in reader:
if row:
self._inputStorage[i] = row
i += 1
self._inputItems = i
self._length.set(i)
sbuf.close()
# logger.debug("setSavedFormInput: %s items" % self._inputItems)
security.declareProtected(ModifyPortalContent, 'clearSavedFormInput')
def clearSavedFormInput(self, **kwargs):
""" convenience method to clear input buffer """
REQUEST = kwargs.get('request', self.REQUEST)
if REQUEST.form.has_key('clearSavedFormInput'):
# we're processing a request from the web;
# check for CSRF
plone.protect.CheckAuthenticator(REQUEST)
plone.protect.PostOnly(REQUEST)
self._migrateStorage()
self._inputStorage.clear()
self._inputItems = 0
self._length.set(0)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getSavedFormInputById')
def getSavedFormInputById(self, id):
""" Return the data stored for record with 'id' """
lst = [field.replace('\r','').replace('\n', r'\n') for field in self._inputStorage[id]]
return lst
security.declareProtected(ModifyPortalContent, 'manage_saveData')
def manage_saveData(self, id, data):
""" Save the data for record with 'id' """
self._migrateStorage()
lst = list()
for i in range(0, len(self.getColumnNames())):
lst.append(getattr(data, 'item-%d' % i, '').replace(r'\n', '\n'))
self._inputStorage[id] = lst
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
security.declareProtected(ModifyPortalContent, 'manage_deleteData')
def manage_deleteData(self, id):
""" Delete the data for record with 'id' """
self._migrateStorage()
del self._inputStorage[id]
self._inputItems -= 1
self._length.change(-1)
self.REQUEST.RESPONSE.redirect(self.absolute_url() + '/view')
def _addDataRow(self, value):
self._migrateStorage()
if isinstance(self._inputStorage, IOBTree):
# 32-bit IOBTree; use a key which is more likely to conflict
# but which won't overflow the key's bits
id = self._inputItems
self._inputItems += 1
else:
# 64-bit LOBTree
id = int(time.time() * 1000)
while id in self._inputStorage: # avoid collisions during testing
id += 1
self._inputStorage[id] = value
self._length.change(1)
security.declareProtected(ModifyPortalContent, 'addDataRow')
def addDataRow(self, value):
""" a wrapper for the _addDataRow method """
self._addDataRow(value)
def onSuccess(self, fields, REQUEST=None, loopstop=False):
"""
saves data.
"""
if LP_SAVE_TO_CANONICAL and not loopstop:
# LinguaPlone functionality:
# check to see if we're in a translated
# form folder, but not the canonical version.
parent = self.aq_parent
if safe_hasattr(parent, 'isTranslation') and \
parent.isTranslation() and not parent.isCanonical():
# look in the canonical version to see if there is
# a matching (by id) save-data adapter.
# If so, call its onSuccess method
cf = parent.getCanonical()
target = cf.get(self.getId())
if target is not None and target.meta_type == 'FormSaveDataAdapter':
target.onSuccess(fields, REQUEST, loopstop=True)
return
from ZPublisher.HTTPRequest import FileUpload
data = []
for f in fields:
if f.isFileField():
file = REQUEST.form.get('%s_file' % f.fgField.getName())
if isinstance(file, FileUpload) and file.filename != '':
file.seek(0)
fdata = file.read()
filename = file.filename
mimetype, enc = guess_content_type(filename, fdata, None)
if mimetype.find('text/') >= 0:
# convert to native eols
fdata = fdata.replace('\x0d\x0a', '\n').replace('\x0a', '\n').replace('\x0d', '\n')
data.append( '%s:%s:%s:%s' % (filename, mimetype, enc, fdata) )
else:
data.append( '%s:%s:%s:Binary upload discarded' % (filename, mimetype, enc) )
else:
data.append( 'NO UPLOAD' )
elif not f.isLabel():
val = REQUEST.form.get(f.fgField.getName(),'')
if not type(val) in StringTypes:
# Zope has marshalled the field into
# something other than a string
val = str(val)
data.append(val)
if self.ExtraData:
for f in self.ExtraData:
if f == 'dt':
data.append( str(DateTime()) )
else:
data.append( getattr(REQUEST, f, '') )
self._addDataRow( data )
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames')
def getColumnNames(self):
"""Returns a list of column names"""
names = [field.getName() for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(f)
return names
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles')
def getColumnTitles(self):
"""Returns a list of column titles"""
names = [field.widget.label for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(self.vocabExtraDataDL().getValue(f, ''))
return names
def _cleanInputForTSV(self, value):
# make data safe to store in tab-delimited format
return str(value).replace('\x0d\x0a', r'\n').replace('\x0a', r'\n').replace('\x0d', r'\n').replace('\t', r'\t')
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_tsv')
def download_tsv(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
filename = self.id
if filename.find('.') < 0:
filename = '%s.tsv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/tab-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
res = "%s\n" % '\t'.join( self.getColumnNames() )
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
for row in self.getSavedFormInput():
res = '%s%s\n' % (res, '\t'.join( [self._cleanInputForTSV(col) for col in row] ))
return res
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download_csv')
def download_csv(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
filename = self.id
if filename.find('.') < 0:
filename = '%s.csv' % filename
header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
RESPONSE.setHeader("Content-Type", 'text/comma-separated-values;charset=%s' % self.getCharset())
if getattr(self, 'UseColumnNames', False):
delimiter = self.csvDelimiter()
res = "%s\n" % delimiter.join( self.getColumnNames() )
if isinstance(res, unicode):
res = res.encode(self.getCharset())
else:
res = ''
return '%s%s' % (res, self.getSavedFormInputForEdit())
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'download')
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
else:
assert format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'rowAsColDict')
def rowAsColDict(self, row, cols):
""" Where row is a data sequence and cols is a column name sequence,
returns a dict of colname:column. This is a convenience method
used in the record view.
"""
colcount = len(cols)
rdict = {}
for i in range(0, len(row)):
if i < colcount:
rdict[cols[i]] = row[i]
else:
rdict['column-%s' % i] = row[i]
return rdict
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'inputAsDictionaries')
def inputAsDictionaries(self):
"""returns saved data as an iterable of dictionaries
"""
cols = self.getColumnNames()
for row in self.getSavedFormInput():
yield self.rowAsColDict(row, cols)
# alias for old mis-naming
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'InputAsDictionaries')
InputAsDictionaries = inputAsDictionaries
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'formatMIME')
def formatMIME(self):
"""MIME format selected for download
"""
format = getattr(self, 'DownloadFormat', 'tsv')
if format == 'tsv':
return 'text/tab-separated-values'
else:
assert format == 'csv', 'Unknown download format'
return 'text/comma-separated-values'
security.declarePrivate('csvDelimiter')
def csvDelimiter(self):
"""Delimiter character for CSV downloads
"""
fgt = getToolByName(self, 'formgen_tool')
return fgt.getCSVDelimiter()
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'itemsSaved')
def itemsSaved(self):
"""Download the saved data
"""
if base_hasattr(self, '_length'):
return self._length()
elif base_hasattr(self, '_inputItems'):
return self._inputItems
else:
return len(self.SavedFormInput)
def vocabExtraDataDL(self):
""" returns vocabulary for extra data """
return DisplayList( (
('dt',
self.translate( msgid='vocabulary_postingdt_text',
domain='ploneformgen',
default='Posting Date/Time')
),
('HTTP_X_FORWARDED_FOR','HTTP_X_FORWARDED_FOR',),
('REMOTE_ADDR','REMOTE_ADDR',),
('HTTP_USER_AGENT','HTTP_USER_AGENT',),
) )
def vocabFormatDL(self):
""" returns vocabulary for format """
return DisplayList( (
('tsv',
self.translate( msgid='vocabulary_tsv_text',
domain='ploneformgen',
default='Tab-Separated Values')
),
('csv',
self.translate( msgid='vocabulary_csv_text',
domain='ploneformgen',
default='Comma-Separated Values')
),
) )
class HBTreeFolder2Base (Persistent):
"""Base for BTree-based folders.
"""
security = ClassSecurityInfo()
manage_options=(
({'label':'Contents', 'action':'manage_main',},
) + Folder.manage_options[1:]
)
security.declareProtected(view_management_screens,
'manage_main')
manage_main = DTMLFile('contents', globals())
_htree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
title = ''
_tree_list = None
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._htree = OOBTree()
self._count = Length()
self._tree_list = PersistentMapping()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name in source.objectIds():
value = source._getOb(name, None)
if value is not None:
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount()
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in HBTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in HBTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self):
"""Checks if the value of self._count disagrees with
len(self.objectIds()). If so, corrects self._count. Returns the
old and new count values. If old==new, no correction was
performed.
"""
old = self._count()
new = len(self.objectIds())
if old != new:
self._count.set(new)
return old, new
security.declareProtected(view_management_screens, 'manage_cleanup')
def manage_cleanup(self):
"""Calls self._cleanup() and reports the result as text.
"""
v = self._cleanup()
path = '/'.join(self.getPhysicalPath())
if v:
return "No damage detected in HBTreeFolder2 at %s." % path
else:
return ("Fixed HBTreeFolder2 at %s. "
"See the log for more details." % path)
def _cleanup(self):
"""Cleans up errors in the BTrees.
Certain ZODB bugs have caused BTrees to become slightly insane.
Fortunately, there is a way to clean up damaged BTrees that
always seems to work: make a new BTree containing the items()
of the old one.
Returns 1 if no damage was detected, or 0 if damage was
detected and fixed.
"""
def hCheck(htree):
"""
Recursively check the btree
"""
check(htree)
for key in htree.keys():
if not htree.has_key(key):
raise AssertionError(
"Missing value for key: %s" % repr(key))
else:
ob = htree[key]
if isinstance(ob, OOBTree):
hCheck(ob)
return 1
from BTrees.check import check
path = '/'.join(self.getPhysicalPath())
try:
return hCheck(self._htree)
except AssertionError:
LOG('HBTreeFolder2', WARNING,
'Detected damage to %s. Fixing now.' % path,
error=sys.exc_info())
try:
self._htree = OOBTree(self._htree) # XXX hFix needed
except:
LOG('HBTreeFolder2', ERROR, 'Failed to fix %s.' % path,
error=sys.exc_info())
raise
else:
LOG('HBTreeFolder2', INFO, 'Fixed %s.' % path)
return 0
def hashId(self, id):
"""Return a tuple of ids
"""
# XXX: why tolerate non-string ids ?
id_list = str(id).split(H_SEPARATOR) # We use '-' as the separator by default
if len(id_list) > 1:
return tuple(id_list)
else:
return [id,]
# try: # We then try int hashing
# id_int = int(id)
# except ValueError:
# return id_list
# result = []
# while id_int:
# result.append(id_int % MAX_OBJECT_PER_LEVEL)
# id_int = id_int / MAX_OBJECT_PER_LEVEL
# result.reverse()
# return tuple(result)
def _getOb(self, id, default=_marker):
"""
Return the named object from the folder.
"""
htree = self._htree
ob = htree
id_list = self.hashId(id)
for sub_id in id_list[0:-1]:
if default is _marker:
ob = ob[sub_id]
else:
ob = ob.get(sub_id, _marker)
if ob is _marker:
return default
if default is _marker:
ob = ob[id]
else:
ob = ob.get(id, _marker)
if ob is _marker:
return default
return ob.__of__(self)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
htree = self._htree
id_list = self.hashId(id)
for idx in xrange(len(id_list) - 1):
sub_id = id_list[idx]
if not htree.has_key(sub_id):
# Create a new level
htree[sub_id] = OOBTree()
if isinstance(sub_id, (int, long)):
tree_id = 0
for id in id_list[:idx+1]:
tree_id = tree_id + id * MAX_OBJECT_PER_LEVEL
else:
tree_id = H_SEPARATOR.join(id_list[:idx+1])
# Index newly created level
self._tree_list[tree_id] = None
htree = htree[sub_id]
if len(id_list) == 1 and not htree.has_key(None):
self._tree_list[None] = None
# set object in subtree
ob_id = id_list[-1]
if htree.has_key(id):
raise KeyError('There is already an item named "%s".' % id)
htree[id] = object
self._count.change(1)
def _delOb(self, id):
"""Remove the named object from the folder.
"""
htree = self._htree
id_list = self.hashId(id)
for sub_id in id_list[0:-1]:
htree = htree[sub_id]
del htree[id]
self._count.change(-1)
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = [listtext0 % pref_rows]
for optID in islice(self.objectIds(), b_start - 1, b_end):
optID = escape(optID)
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect(
'%s/%s/manage_workspace' % (
self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information,
'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information,
'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
htree = self._htree
id_list = self.hashId(id)
for sub_id in id_list[0:-1]:
if not isinstance(htree, OOBTree):
return 0
if not htree.has_key(sub_id):
return 0
htree = htree[sub_id]
if not htree.has_key(id):
return 0
return 1
# Work around for the performance regression introduced in Zope 2.12.23.
# Otherwise, we use superclass' __contains__ implementation, which uses
# objectIds, which is inefficient in HBTreeFolder2 to lookup a single key.
__contains__ = has_key
def _htree_iteritems(self, min=None):
# BUG: Due to bad design of HBTreeFolder2, buckets other than the root
# one must not contain both buckets & leafs. Otherwise, this method
# fails.
h = self._htree
recurse_stack = []
try:
for sub_id in min and self.hashId(min) or ('',):
if recurse_stack:
i.next()
if type(h) is not OOBTree:
break
id += H_SEPARATOR + sub_id
if type(h.itervalues().next()) is not OOBTree:
sub_id = id
else:
id = sub_id
i = h.iteritems(sub_id)
recurse_stack.append(i)
h = h[sub_id]
except (KeyError, StopIteration):
pass
while recurse_stack:
i = recurse_stack.pop()
try:
while 1:
id, h = i.next()
if type(h) is OOBTree:
recurse_stack.append(i)
i = h.iteritems()
else:
yield id, h
except StopIteration:
pass
security.declareProtected(access_contents_information,
'treeIds')
def treeIds(self, base_id=None):
""" Return a list of subtree ids
"""
tree = self._getTree(base_id=base_id)
return [k for k, v in self._htree.items() if isinstance(v, OOBTree)]
def _getTree(self, base_id):
""" Return the tree wich has the base_id
"""
htree = self._htree
id_list = self.hashId(base_id)
for sub_id in id_list:
if not isinstance(htree, OOBTree):
return None
if not htree.has_key(sub_id):
raise IndexError, base_id
htree = htree[sub_id]
return htree
def _getTreeIdList(self, htree=None):
""" recursively build a list of btree ids
"""
if htree is None:
htree = self._htree
btree_list = []
else:
btree_list = []
for obj_id in htree.keys():
obj = htree[obj_id]
if isinstance(obj, OOBTree):
btree_list.extend(["%s-%s"%(obj_id, x) for x in self._getTreeIdList(htree=obj)])
btree_list.append(obj_id)
return btree_list
security.declareProtected(access_contents_information,
'getTreeIdList')
def getTreeIdList(self, htree=None):
""" Return list of all tree ids
"""
if self._tree_list is None or len(self._tree_list.keys()) == 0:
tree_list = self._getTreeIdList(htree=htree)
self._tree_list = PersistentMapping()
for tree in tree_list:
self._tree_list[tree] = None
return sorted(self._tree_list.keys())
def _checkObjectId(self, ids):
""" test id is not in btree id list
"""
base_id, obj_id = ids
if base_id is not None:
obj_id = "%s%s%s" %(base_id, H_SEPARATOR, obj_id)
return not self._tree_list.has_key(obj_id)
security.declareProtected(access_contents_information,
'objectValues')
def objectValues(self, base_id=_marker):
return HBTreeObjectValues(self, base_id)
security.declareProtected(access_contents_information,
'objectIds')
def objectIds(self, base_id=_marker):
return HBTreeObjectIds(self, base_id)
security.declareProtected(access_contents_information,
'objectItems')
def objectItems(self, base_id=_marker):
# Returns a list of (id, subobject) tuples of the current object.
# If 'spec' is specified, returns only objects whose meta_type match
# 'spec'
return HBTreeObjectItems(self, base_id)
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information,
'objectIds_d')
def objectIds_d(self, t=None):
return dict.fromkeys(self.objectIds(t), 1)
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self, id, object, roles=None, user=None, set_owner=1):
v=self._checkId(id)
if v is not None: id=v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
self._setOb(id, object)
object = self._getOb(id)
if set_owner:
object.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if hasattr(object, '__ac_local_roles__'):
if object.__ac_local_roles__ is None:
user=getSecurityManager().getUser()
if user is not None:
userid=user.getId()
if userid is not None:
object.manage_setLocalRoles(userid, ['Owner'])
object.manage_afterAdd(object, self)
return id
def _delObject(self, id, dp=1):
object = self._getOb(id)
try:
object.manage_beforeDelete(object, self)
except BeforeDeleteException, ob:
raise
except ConflictError:
raise
class HBTreeFolder2Base (Persistent):
"""Base for BTree-based folders.
BUG: Due to wrong design, we can't store 2 objects <A> and <A>-<B>
where <A> does not contain '-'. We detect conflicts at the
root level using 'type(ob) is OOBTree'
"""
security = ClassSecurityInfo()
manage_options=(
({'label':'Contents', 'action':'manage_main',},
) + Folder.manage_options[1:]
)
security.declareProtected(view_management_screens,
'manage_main')
manage_main = DTMLFile('contents', globals())
_htree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._htree = OOBTree()
self._count = Length()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name, value in source.objectItems():
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self, dry_run=0):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount(dry_run)
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in HBTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in HBTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self, dry_run=0):
"""Checks if the value of self._count disagrees with the content of
the htree. If so, corrects self._count. Returns the old and new count
values. If old==new, no correction was performed.
"""
old = self._count()
new = sum(1 for x in self._htree_iteritems())
if old != new and not dry_run:
self._count.set(new)
return old, new
def hashId(self, id):
return id.split(H_SEPARATOR)
def _htree_get(self, id):
id_list = self.hashId(id)
if len(id_list) == 1:
ob = self._htree[id]
if type(ob) is OOBTree:
raise KeyError
else:
ob = self._htree[id_list.pop(0)]
if type(ob) is not OOBTree:
raise KeyError
id_list[-1] = id
for sub_id in id_list:
ob = ob[sub_id]
return ob
def _getOb(self, id, default=_marker):
"""Return the named object from the folder
"""
try:
return self._htree_get(id).__of__(self)
except KeyError:
if default is _marker:
raise KeyError(id)
return default
def __getitem__(self, id):
try:
return self._htree_get(id).__of__(self)
except KeyError:
raise KeyError(id)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
if type(object) is OOBTree:
raise ValueError('HBTreeFolder2 can not store OOBTree objects')
htree = self._htree
for sub_id in self.hashId(id)[:-1]:
try:
htree = htree[sub_id]
except KeyError:
htree[sub_id] = htree = OOBTree()
continue
if type(htree) is not OOBTree:
assert self._htree[sub_id] is htree, (htree, id)
raise KeyError('There is already an item whose id is %r' % sub_id)
if htree.has_key(id):
raise KeyError('There is already an item named %r.' % id)
htree[id] = object
self._count.change(1)
def _delOb(self, id):
"""Remove the named object from the folder.
"""
htree = self._htree
h = []
for sub_id in self.hashId(id)[:-1]:
h.append((htree, sub_id))
htree = htree.get(sub_id)
if type(htree) is not OOBTree:
raise KeyError(id)
if type(htree[id]) is OOBTree:
raise KeyError(id)
del htree[id]
self._count.change(-1)
while h and not htree:
htree, sub_id = h.pop()
del htree[sub_id]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = [listtext0 % pref_rows]
for optID in islice(self.objectIds(), b_start - 1, b_end):
optID = escape(optID)
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect(
'%s/%s/manage_workspace' % (
self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information,
'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information,
'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
try:
self._htree_get(id)
except KeyError:
return 0
return 1
# Work around for the performance regression introduced in Zope 2.12.23.
# Otherwise, we use superclass' __contains__ implementation, which uses
# objectIds, which is inefficient in HBTreeFolder2 to lookup a single key.
__contains__ = has_key
def _htree_iteritems(self, min=None):
# BUG: Due to bad design of HBTreeFolder2, buckets other than the root
# one must not contain both buckets & leafs. Otherwise, this method
# fails.
h = self._htree
recurse_stack = []
try:
for sub_id in self.hashId(min) if min else ('',):
if recurse_stack:
i.next()
if type(h) is not OOBTree:
break
id += H_SEPARATOR + sub_id
if type(h.itervalues().next()) is not OOBTree:
sub_id = id
else:
id = sub_id
i = h.iteritems(sub_id)
recurse_stack.append(i)
h = h[sub_id]
except (KeyError, StopIteration):
pass
while recurse_stack:
i = recurse_stack.pop()
try:
while 1:
id, h = i.next()
if type(h) is OOBTree:
recurse_stack.append(i)
i = h.iteritems()
else:
yield id, h
except StopIteration:
pass
security.declareProtected(access_contents_information,
'getTreeIdList')
def getTreeIdList(self, htree=None):
""" Return list of all tree ids
"""
r = []
s = [(None, self._htree.iteritems())]
while s:
base_id, items = s.pop()
if base_id:
for k, v in items:
if type(v) is not OOBTree:
r.append(base_id)
# As an optimization, and because _htree_iteritems does not
# support mixed buckets except at the root, we consider that
# this one only contains leafs.
break
s.append((base_id + H_SEPARATOR + k, v.iteritems()))
else:
for k, v in items:
if type(v) is not OOBTree:
r.append(base_id)
for k, v in items:
if type(v) is OOBTree:
s.append((k, v.iteritems()))
break
s.append((k, v.iteritems()))
r.sort()
return r
security.declareProtected(access_contents_information,
'objectValues')
def objectValues(self, base_id=_marker):
return HBTreeObjectValues(self, base_id)
security.declareProtected(access_contents_information,
'objectIds')
def objectIds(self, base_id=_marker):
return HBTreeObjectIds(self, base_id)
security.declareProtected(access_contents_information,
'objectItems')
def objectItems(self, base_id=_marker):
# Returns a list of (id, subobject) tuples of the current object.
return HBTreeObjectItems(self, base_id)
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information,
'objectIds_d')
def objectIds_d(self, t=None):
return dict.fromkeys(self.objectIds(t), 1)
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self, id, object, roles=None, user=None, set_owner=1):
v=self._checkId(id)
if v is not None: id=v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
self._setOb(id, object)
object = self._getOb(id)
if set_owner:
object.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if hasattr(object, '__ac_local_roles__'):
if object.__ac_local_roles__ is None:
user=getSecurityManager().getUser()
if user is not None:
userid=user.getId()
if userid is not None:
object.manage_setLocalRoles(userid, ['Owner'])
object.manage_afterAdd(object, self)
return id
def _delObject(self, id, dp=1):
object = self._getOb(id)
try:
object.manage_beforeDelete(object, self)
except BeforeDeleteException, ob:
raise
except ConflictError:
raise
class HBTreeFolder2Base (Persistent):
"""Base for BTree-based folders.
BUG: Due to wrong design, we can't store 2 objects <A> and <A>-<B>
where <A> does not contain '-'. We detect conflicts at the
root level using 'type(ob) is OOBTree'
"""
security = ClassSecurityInfo()
manage_options=(
({'label':'Contents', 'action':'manage_main',},
) + Folder.manage_options[1:]
)
security.declareProtected(view_management_screens,
'manage_main')
manage_main = DTMLFile('contents', globals())
_htree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
title = ''
def __init__(self, id=None):
if id is not None:
self.id = id
self._initBTrees()
def _initBTrees(self):
self._htree = OOBTree()
self._count = Length()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
"""
for name, value in source.objectItems():
self._setOb(name, aq_base(value))
security.declareProtected(view_management_screens, 'manage_fixCount')
def manage_fixCount(self, dry_run=0):
"""Calls self._fixCount() and reports the result as text.
"""
old, new = self._fixCount(dry_run)
path = '/'.join(self.getPhysicalPath())
if old == new:
return "No count mismatch detected in HBTreeFolder2 at %s." % path
else:
return ("Fixed count mismatch in HBTreeFolder2 at %s. "
"Count was %d; corrected to %d" % (path, old, new))
def _fixCount(self, dry_run=0):
"""Checks if the value of self._count disagrees with the content of
the htree. If so, corrects self._count. Returns the old and new count
values. If old==new, no correction was performed.
"""
old = self._count()
new = sum(1 for x in self._htree_iteritems())
if old != new and not dry_run:
self._count.set(new)
return old, new
def hashId(self, id):
return id.split(H_SEPARATOR)
def _htree_get(self, id):
id_list = self.hashId(id)
if len(id_list) == 1:
ob = self._htree[id]
if type(ob) is OOBTree:
raise KeyError
else:
ob = self._htree[id_list.pop(0)]
if type(ob) is not OOBTree:
raise KeyError
id_list[-1] = id
for sub_id in id_list:
ob = ob[sub_id]
return ob
def _getOb(self, id, default=_marker):
"""Return the named object from the folder
"""
try:
return self._htree_get(id).__of__(self)
except KeyError:
if default is _marker:
raise KeyError(id)
return default
def __getitem__(self, id):
try:
return self._htree_get(id).__of__(self)
except KeyError:
raise KeyError(id)
def _setOb(self, id, object):
"""Store the named object in the folder.
"""
if type(object) is OOBTree:
raise ValueError('HBTreeFolder2 can not store OOBTree objects')
htree = self._htree
for sub_id in self.hashId(id)[:-1]:
try:
htree = htree[sub_id]
except KeyError:
htree[sub_id] = htree = OOBTree()
continue
if type(htree) is not OOBTree:
assert self._htree[sub_id] is htree, (htree, id)
raise KeyError('There is already an item whose id is %r' % sub_id)
if htree.has_key(id):
raise KeyError('There is already an item named %r.' % id)
htree[id] = object
self._count.change(1)
def _delOb(self, id):
"""Remove the named object from the folder.
"""
htree = self._htree
h = []
for sub_id in self.hashId(id)[:-1]:
h.append((htree, sub_id))
htree = htree.get(sub_id)
if type(htree) is not OOBTree:
raise KeyError(id)
if type(htree[id]) is OOBTree:
raise KeyError(id)
del htree[id]
self._count.change(-1)
while h and not htree:
htree, sub_id = h.pop()
del htree[sub_id]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
"""Return a structure for a page template to show the list of objects.
"""
if REQUEST is None:
REQUEST = {}
pref_rows = int(REQUEST.get('dtpref_rows', 20))
b_start = int(REQUEST.get('b_start', 1))
b_count = int(REQUEST.get('b_count', 1000))
b_end = b_start + b_count - 1
url = self.absolute_url() + '/manage_main'
count = self.objectCount()
if b_end < count:
next_url = url + '?b_start=%d' % (b_start + b_count)
else:
b_end = count
next_url = ''
if b_start > 1:
prev_url = url + '?b_start=%d' % max(b_start - b_count, 1)
else:
prev_url = ''
formatted = [listtext0 % pref_rows]
for optID in islice(self.objectIds(), b_start - 1, b_end):
optID = escape(optID)
formatted.append(listtext1 % (escape(optID, quote=1), optID))
formatted.append(listtext2)
return {'b_start': b_start, 'b_end': b_end,
'prev_batch_url': prev_url,
'next_batch_url': next_url,
'formatted_list': ''.join(formatted)}
security.declareProtected(view_management_screens,
'manage_object_workspace')
def manage_object_workspace(self, ids=(), REQUEST=None):
'''Redirects to the workspace of the first object in
the list.'''
if ids and REQUEST is not None:
REQUEST.RESPONSE.redirect(
'%s/%s/manage_workspace' % (
self.absolute_url(), quote(ids[0])))
else:
return self.manage_main(self, REQUEST)
security.declareProtected(access_contents_information,
'tpValues')
def tpValues(self):
"""Ensures the items don't show up in the left pane.
"""
return ()
security.declareProtected(access_contents_information,
'objectCount')
def objectCount(self):
"""Returns the number of items in the folder."""
return self._count()
security.declareProtected(access_contents_information, 'has_key')
def has_key(self, id):
"""Indicates whether the folder has an item by ID.
"""
try:
self._htree_get(id)
except KeyError:
return 0
return 1
# Work around for the performance regression introduced in Zope 2.12.23.
# Otherwise, we use superclass' __contains__ implementation, which uses
# objectIds, which is inefficient in HBTreeFolder2 to lookup a single key.
__contains__ = has_key
def _htree_iteritems(self, min=None):
# BUG: Due to bad design of HBTreeFolder2, buckets other than the root
# one must not contain both buckets & leafs. Otherwise, this method
# fails.
h = self._htree
recurse_stack = []
try:
for sub_id in self.hashId(min) if min else ('',):
if recurse_stack:
i.next()
if type(h) is not OOBTree:
break
id += H_SEPARATOR + sub_id
if type(h.itervalues().next()) is not OOBTree:
sub_id = id
else:
id = sub_id
i = h.iteritems(sub_id)
recurse_stack.append(i)
h = h[sub_id]
except (KeyError, StopIteration):
pass
while recurse_stack:
i = recurse_stack.pop()
try:
while 1:
id, h = i.next()
if type(h) is OOBTree:
recurse_stack.append(i)
i = h.iteritems()
else:
yield id, h
except StopIteration:
pass
security.declareProtected(access_contents_information,
'getTreeIdList')
def getTreeIdList(self, htree=None):
""" Return list of all tree ids
"""
r = []
s = [(None, self._htree.iteritems())]
while s:
base_id, items = s.pop()
if base_id:
for k, v in items:
if type(v) is not OOBTree:
r.append(base_id)
# As an optimization, and because _htree_iteritems does not
# support mixed buckets except at the root, we consider that
# this one only contains leafs.
break
s.append((base_id + H_SEPARATOR + k, v.iteritems()))
else:
for k, v in items:
if type(v) is not OOBTree:
r.append(base_id)
for k, v in items:
if type(v) is OOBTree:
s.append((k, v.iteritems()))
break
s.append((k, v.iteritems()))
r.sort()
return r
security.declareProtected(access_contents_information,
'objectValues')
def objectValues(self, base_id=_marker):
return HBTreeObjectValues(self, base_id)
security.declareProtected(access_contents_information,
'objectIds')
def objectIds(self, base_id=_marker):
return HBTreeObjectIds(self, base_id)
security.declareProtected(access_contents_information,
'objectItems')
def objectItems(self, base_id=_marker):
# Returns a list of (id, subobject) tuples of the current object.
return HBTreeObjectItems(self, base_id)
# superValues() looks for the _objects attribute, but the implementation
# would be inefficient, so superValues() support is disabled.
_objects = ()
security.declareProtected(access_contents_information,
'objectIds_d')
def objectIds_d(self, t=None):
return dict.fromkeys(self.objectIds(t), 1)
def _checkId(self, id, allow_dup=0):
if not allow_dup and self.has_key(id):
raise BadRequestException, ('The id "%s" is invalid--'
'it is already in use.' % id)
def _setObject(self, id, object, roles=None, user=None, set_owner=1):
v=self._checkId(id)
if v is not None: id=v
# If an object by the given id already exists, remove it.
if self.has_key(id):
self._delObject(id)
self._setOb(id, object)
object = self._getOb(id)
if set_owner:
object.manage_fixupOwnershipAfterAdd()
# Try to give user the local role "Owner", but only if
# no local roles have been set on the object yet.
if hasattr(object, '__ac_local_roles__'):
if object.__ac_local_roles__ is None:
user=getSecurityManager().getUser()
if user is not None:
userid=user.getId()
if userid is not None:
object.manage_setLocalRoles(userid, ['Owner'])
object.manage_afterAdd(object, self)
return id
def _delObject(self, id, dp=1):
object = self._getOb(id)
try:
object.manage_beforeDelete(object, self)
except BeforeDeleteException, ob:
raise
except ConflictError:
raise
class FormXLSSaveDataAdapter(ATCTFolder, FormActionAdapter):
"""A form action adapter that will save form input data and
return it in XLS format.
Based on SaveDataAdapter
"""
implements(IFormXLSSaveDataAdapter)
schema = FormAdapterSchema.copy() + atapi.Schema((
atapi.BooleanField("storeAttachments",
required=False,
default=True,
widget=atapi.BooleanWidget(
label = _(u'label_storeAttachments_text', default=u"Store attachments?"),
description = _(u'help_storeAttachments_text', default=u"If checked, FileFields content will be saved inside this save adapter. "
u"Link to the attachment will be part of XLS file. "
u"Please note, if unchecked, attachments may be lost in the Universe."),
),
),
atapi.LinesField('ExtraData',
widget=atapi.MultiSelectionWidget(
label=_(u'label_savedataextra_text', default='Extra Data'),
description=_(u'help_savedataextra_text', default=u"""
Pick any extra data you'd like saved with the form input.
"""),
format='checkbox',
),
vocabulary = 'vocabExtraDataDL',
),
atapi.BooleanField("UseColumnNames",
required=False,
searchable=False,
widget=atapi.BooleanWidget(
label = _(u'label_usecolumnnames_text', default=u"Include Column Names"),
description = _(u'help_usecolumnnames_text', default=u"Do you wish to have column names on the first line of downloaded input?"),
),
),
))
schema.moveField('execCondition', pos='bottom')
security = ClassSecurityInfo()
def _setupStorage(self):
set_up = base_hasattr(self, '_inputStorage') and \
base_hasattr(self, '_inputItems') and \
base_hasattr(self, '_length')
if not set_up:
self._inputStorage = SavedDataBTree()
self._inputItems = 0
self._length = Length()
security.declarePrivate('clear')
def clear(self):
self._inputStorage.clear()
self._inputItems = 0
self._length.set(0)
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'listData')
def listData(self):
self._setupStorage() # be sure we have _inputStorage
for item in self._inputStorage.values():
yield item
def _addDataRow(self, value):
self._setupStorage()
if isinstance(self._inputStorage, IOBTree):
# 32-bit IOBTree; use a key which is more likely to conflict
# but which won't overflow the key's bits
id = self._inputItems
self._inputItems += 1
else:
# 64-bit LOBTree
id = int(time.time() * 1000)
while id in self._inputStorage: # avoid collisions during testing
id += 1
self._inputStorage[id] = value
self._length.change(1)
security.declareProtected(ModifyPortalContent, 'addDataRow')
def addDataRow(self, value):
""" a wrapper for the _addDataRow method """
self._addDataRow(value)
def onSuccess(self, fields, REQUEST=None, loopstop=False):
"""
saves data.
"""
if LP_SAVE_TO_CANONICAL and not loopstop:
# LinguaPlone functionality:
# check to see if we're in a translated
# form folder, but not the canonical version.
parent = self.aq_parent
if safe_hasattr(parent, 'isTranslation') and \
parent.isTranslation() and not parent.isCanonical():
# look in the canonical version to see if there is
# a matching (by id) save-data adapter.
# If so, call its onSuccess method
cf = parent.getCanonical()
target = cf.get(self.getId())
if target is not None and target.meta_type == 'FormXLSSaveDataAdapter':
target.onSuccess(fields, REQUEST, loopstop=True)
return
from ZPublisher.HTTPRequest import FileUpload
data = dict()
for f in fields:
name = f.fgField.getName()
if f.isFileField():
file = REQUEST.form.get('%s_file' % name)
if isinstance(file, FileUpload) and file.filename != '':
if self.getStoreAttachments():
file.seek(0)
fdata = file.read()
filename = file.filename
mimetype, enc = guess_content_type(filename, fdata, None)
# create File object
_id = get_safe_id(REQUEST, self, filename)
obj = _createObjectByType('FormXLSSaveDataFile', self, _id)
file.seek(0)
obj.setTitle(filename)
obj.setFile(file)
obj.unmarkCreationFlag()
obj.reindexObject()
url = obj.absolute_url()
data[name] = url
else:
data[name] = 'DISCARDED'
else:
data[name] = 'NO UPLOAD'
elif not f.isLabel():
val = f.htmlValue(REQUEST)
data[name] = val
if self.ExtraData:
for f in self.ExtraData:
name = 'extra_' + f
if f == 'dt':
data[name] = str(DateTime())
else:
if f in REQUEST:
data[name] = REQUEST.get(f)
self._addDataRow( data )
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnNames')
def getColumnNames(self):
"""Returns a list of column names"""
names = [field.getName() for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(f)
return names
security.declareProtected(DOWNLOAD_SAVED_PERMISSION, 'getColumnTitles')
def getColumnTitles(self):
"""Returns a list of column titles"""
names = [field.widget.label for field in self.fgFields(displayOnly=True)]
for f in self.ExtraData:
names.append(self.vocabExtraDataDL().getValue(f, ''))
return names
def vocabExtraDataDL(self):
""" returns vocabulary for extra data """
return atapi.DisplayList( (
('dt',
self.translate( msgid='vocabulary_postingdt_text',
domain='ploneformgen',
default='Posting Date/Time')
),
('HTTP_X_FORWARDED_FOR','HTTP_X_FORWARDED_FOR',),
('REMOTE_ADDR','REMOTE_ADDR',),
('HTTP_USER_AGENT','HTTP_USER_AGENT',),
) )
