class List(Base):
def __init__(self, parent):
Base.__init__(self, parent)
self.__pers_list = PersistentList()
def show(self):
for x in self:
print x
def __repr__(self):
return self.__pers_list.__repr__()
def __getitem__(self, key):
return self.__pers_list.__getitem__(key)
def __len__(self):
return self.__pers_list.__len__()
def __getattr__(self, name):
if name in [
"append", "extend", "insert", "pop", "remove", "reverse",
"sort"
]:
return getattr(self.__pers_list, name)
return Base.__getattr__(self, name)
class Profile(Persistent):
FolderClass = Folder
def __init__(self, folder_dir):
self._dir = folder_dir
self.classifier = PBayes()
self.hams = PersistentList()
self.spams = PersistentList()
def add_ham(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.hams.append(f)
def add_spam(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.spams.append(f)
def update(self):
"""Update classifier from current folder contents."""
changed1 = self._update(self.hams, False)
changed2 = self._update(self.spams, True)
## if changed1 or changed2:
## self.classifier.update_probabilities()
get_transaction().commit()
log("updated probabilities")
def _update(self, folders, is_spam):
changed = False
for f in folders:
log("update from %s" % f.path)
added, removed = f.read()
if added:
log("added %d" % len(added))
if removed:
log("removed %d" % len(removed))
get_transaction().commit()
if not (added or removed):
continue
changed = True
# It's important not to commit a transaction until
# after update_probabilities is called in update().
# Otherwise some new entries will cause scoring to fail.
for msg in added.keys():
self.classifier.learn(tokenize(msg), is_spam)
del added
get_transaction().commit(1)
log("learned")
for msg in removed.keys():
self.classifier.unlearn(tokenize(msg), is_spam)
if removed:
log("unlearned")
del removed
get_transaction().commit(1)
return changed
class Profile(Persistent):
FolderClass = Folder
def __init__(self, folder_dir):
self._dir = folder_dir
self.classifier = PBayes()
self.hams = PersistentList()
self.spams = PersistentList()
def add_ham(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.hams.append(f)
def add_spam(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.spams.append(f)
def update(self):
"""Update classifier from current folder contents."""
changed1 = self._update(self.hams, False)
changed2 = self._update(self.spams, True)
## if changed1 or changed2:
## self.classifier.update_probabilities()
get_transaction().commit()
log("updated probabilities")
def _update(self, folders, is_spam):
changed = False
for f in folders:
log("update from %s" % f.path)
added, removed = f.read()
if added:
log("added %d" % len(added))
if removed:
log("removed %d" % len(removed))
get_transaction().commit()
if not (added or removed):
continue
changed = True
# It's important not to commit a transaction until
# after update_probabilities is called in update().
# Otherwise some new entries will cause scoring to fail.
for msg in added.keys():
self.classifier.learn(tokenize(msg), is_spam)
del added
get_transaction().commit(1)
log("learned")
for msg in removed.keys():
self.classifier.unlearn(tokenize(msg), is_spam)
if removed:
log("unlearned")
del removed
get_transaction().commit(1)
return changed
def __init__(self):
self._v_current_card = 0
self._v_text = None
self._v_cardNumber = None
self.last_id = 0
self.card_list = PersistentList()
self.to_be_indexed = PersistentList()
self.card_store = OOBTree.OOBTree()
self.hint_store = OOBTree.OOBTree()
self.CreateCard(at_end=1, and_set=0)
def __init__(self):
"""Initialize (singleton!) tool object."""
# user configurable
self.dictServers = {}
self.secProcessWindows = {}
self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW
# unknown to the user
self.pendingRequests = PersistentList([])
self.processingRequests = PersistentList([])
log('__init__ completed.') # this no workeee???
def update_dataflow_uris(root, commit=False):
candidates = filter_objects(root, filter_level=0)
counter = 0
changes_log.info('DATAFLOW URIS UPDATES')
transaction.savepoint()
for obj in candidates:
dataflow_uris = getattr(obj, 'dataflow_uris', None)
dataflow_uri = ''
corrected_uris = PersistentList()
corrected_uri = ''
message = ''
message += ('{type:21}: {url}'.format(
type=(obj.meta_type[:17]+'... ' if len(obj.meta_type)>20
else obj.meta_type),
url=obj.absolute_url_path()))
for uri in getattr(obj, 'dataflow_uris', []):
corrected_uris.append(uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu'))
if not(getattr(obj, 'dataflow_uris', None) is None):
obj.dataflow_uris = corrected_uris
if not dataflow_uris == corrected_uris:
message += '\n{attr:21}: {before} -> {after}'.format(
attr='dataflow_uris',
before=dataflow_uris,
after=obj.dataflow_uris)
if getattr(obj, 'dataflow_uri', None):
dataflow_uri = obj.dataflow_uri
corrected_uri = dataflow_uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu')
obj.dataflow_uri = corrected_uri
if not dataflow_uri == corrected_uri:
message += '\n{attr:21}: {before} -> {after}'.format(
attr='dataflow_uri',
before=dataflow_uri,
after=corrected_uri)
if getattr(obj, 'country', None):
country_uri = obj.country
corrected_country = country_uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu')
obj.country = corrected_country
if not country_uri == corrected_country:
message += '\n{attr:21}: {before} -> {after}'.format(
attr='country',
before=country_uri,
after=corrected_country)
if not obj._p_changed:
message+="\nno changes made"
message+="\n"
changes_log.info(message)
counter+=1
if counter % 1000 == 0:
transaction.savepoint()
if commit:
transaction.commit()
changes_log.info('ALL CHANGES COMMITED TO ZODB!')
def testNewFolderCriterias(self):
"""
testes that the import_criterias for a fold changed after
being modifieded (folder bg)
"""
duplicates_criteria = {
'BookReference': PersistentList(('authors', 'title'))
}
self.second_bf = self.getEmptyBibFolder('second_bib_folder')
old_criterias = self.bf.getCriterias()
sec_old_criterias = self.second_bf.getCriterias()
for bib_type in duplicates_criteria.keys():
self.bf.setCriteriasForType(bib_type,
duplicates_criteria[bib_type])
new_criterias = self.bf.getCriterias()
sec_new_criterias = self.second_bf.getCriterias()
self.assertNotEquals(old_criterias, new_criterias,
'Criterias have not been changed')
self.assertEquals(sec_old_criterias, sec_new_criterias,
'Criterias have been changed')
class Profile(Persistent):
FolderClass = Folder
def __init__(self, folder_dir):
self._dir = folder_dir
self.classifier = PBayes()
self.hams = PersistentList()
self.spams = PersistentList()
def add_ham(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.hams.append(f)
def add_spam(self, folder):
p = os.path.join(self._dir, folder)
f = self.FolderClass(p)
self.spams.append(f)
def update(self):
"""Update classifier from current folder contents."""
changed1 = self._update(self.hams, False)
changed2 = self._update(self.spams, True)
get_transaction().commit()
log("updated probabilities")
def _update(self, folders, is_spam):
changed = False
for f in folders:
log("update from %s" % f.path)
added, removed = f.read()
if added:
log("added %d" % len(added))
if removed:
log("removed %d" % len(removed))
get_transaction().commit()
if not (added or removed):
continue
changed = True
for msg in added.keys():
self.classifier.learn(tokenize(msg), is_spam)
del added
get_transaction().commit(1)
log("learned")
for msg in removed.keys():
self.classifier.unlearn(tokenize(msg), is_spam)
if removed:
log("unlearned")
del removed
get_transaction().commit(1)
return changed
def make_config_persistent(kwargs):
""" iterates on the given dictionnary and replace list by persistent list,
dictionary by persistent mapping.
"""
for key, value in kwargs.items():
if type(value) == type({}):
p_value = PersistentMapping(value)
kwargs[key] = p_value
elif type(value) in (type(()), type([])):
p_value = PersistentList(value)
kwargs[key] = p_value
def test_persistent_types_buglets(self):
l = PersistentList([1, 2, 3])
self.assertTrue(isinstance(l, PersistentList))
self.assertFalse(isinstance(l, list)) # dangerous
self.assertTrue(isinstance(l[:], PersistentList))
d = PersistentMapping({1: 2})
self.assertTrue(isinstance(d, PersistentMapping))
self.assertFalse(isinstance(d, dict)) # dangerous
self.assertTrue(isinstance(d.copy(), PersistentMapping))
def getSelectedCriteria(self, bib_type=None):
# migrate CMFBAT v0.8 duplicates engine, mending a linguistic fault-pas
if shasattr(self, 'imp_criterias'):
print 'CMFBibliographyAT: performing duplicates engine property update - v0.8 -> v0.9 (getSelectedCriteria of %s)' % '/'.join(
self.getId())
self.duplicates_criteria = PersistentMapping()
self.duplicates_criteria = copy.deepcopy(self.imp_criterias)
self.criteriaUpdated = self.criteriasUpdated
try:
delattr(self, 'imp_criterias')
except:
pass
try:
delattr(self, 'criteriasUpdated')
except:
pass
# first call? initialize self.duplicates_criteria
bib_tool = getToolByName(self, 'portal_bibliography')
if not shasattr(self,
'duplicates_criteria') or not self.duplicates_criteria:
if self.getId() == bib_tool.getId():
for reference_type in bib_tool.getReferenceTypes():
self.duplicates_criteria[reference_type] = PersistentList(
self._default_duplicates_criteria)
self.criteriaUpdated = True
else:
self.duplicates_criteria = bib_tool.getSelectedCriteria()
self.criteriaUpdated = True
if not shasattr(self, '_criteria') or not self._criteria:
self.initCriteria()
# make sure, our selected criteria are in sync with available criteria
duplicates_criteria = {}
for key in self._criteria.keys():
duplicates_criteria[key] = [
criterion for criterion in self._criteria[key]
if self.duplicates_criteria.has_key(key) and (
criterion in self.duplicates_criteria[key])
]
if bib_type:
try:
duplicates_criteria[bib_type]
except KeyError:
return False
return duplicates_criteria[bib_type]
else:
return duplicates_criteria
class QueueTool(UniqueObject, SimpleItem):
"""
Tool for storage of Print files
"""
id = 'queue_tool'
meta_type = 'Queue Tool'
manage_options = (({'label':'Overview', 'action':'manage_overview'},
{'label':'Configure', 'action':'manage_configure'}
)+ SimpleItem.manage_options
)
manage_overview = PageTemplateFile('zpt/manage_overview_queue_tool.zpt', globals())
manage_configure = PageTemplateFile('zpt/manage_configure_queue_tool.zpt', globals())
security = AccessControl.ClassSecurityInfo()
# set default time window to 1 hour
DEFAULT_PROCESS_TIME_WINDOW = 3600
def __init__(self):
"""Initialize (singleton!) tool object."""
# user configurable
self.dictServers = {}
self.secProcessWindows = {}
self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW
# unknown to the user
self.pendingRequests = PersistentList([])
self.processingRequests = PersistentList([])
log('__init__ completed.') # this no workeee???
# Dependency checkers
def printtool_dep(self, request, dep):
"""check for the existence of the given filetype instance (objectId, Version, extenstion) newer than the given timestamp"""
pt = getToolByName(self,'rhaptos_print')
depkey,deptype,depaction,depdetail = dep
ptf_params = (depdetail['objectId'],depdetail['version'],depdetail['extension'])
if pt.doesFileExist(*ptf_params):
mod_date = DateTime(pt.getModificationDate(*ptf_params))
if mod_date > depdetail['newer']:
return None
return dep
def url_dep(self, request, dep):
"""check for bits available from the given url, with a given mimetype and newer than the given timestamp"""
pass
def file_dep(self, request, dep):
"""check for the existence if a file at the given path, that is newer than the given timestamp"""
depkey,deptype,depaction,depdetail = dep
try:
fstat = os.stat(depdetail['path'])
except OSError:
return dep
if DateTime(fstat.st_ctime) > depdetail['newer']:
return None
return dep
security.declareProtected(ManagePermission, 'manage_queue_tool')
def manage_queue_tool(self, dictServers={}, dictProcessWindows={}, secDefaultProcessWindow=DEFAULT_PROCESS_TIME_WINDOW):
"""
Post creation configuration. See manage_configure_queue_tool.zpt
"""
self.dictServers = eval(dictServers) # string not dictionary returned
self.secProcessWindows = eval(dictProcessWindows) # string not dictionary returned
try:
self.secDefaultProcessWindow = int(secDefaultProcessWindow)
except ValueError:
self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW
security.declareProtected(ManagePermission, 'add')
def add(self, key, dictParams, callbackrequesthandler, priority=1):
"""
Add a request to the Pending Queue.
"""
# add() acquires mutex lock. caller is responsible for commiting the transaction.
mutex.acquire()
try:
iListIndex = self.find(key, self.pendingRequests)
if iListIndex is not None:
# remove duplicate
del self.pendingRequests[iListIndex]
dictRequest = dictParams.copy()
dictRequest['key'] = key
dictRequest['requestHandler'] = callbackrequesthandler
dictRequest['timeRequestMade'] = datetime.now()
dictRequest['priority'] = priority
# Walk list, insert immediately before first entry that is lower priority (higher value)
for i,req in enumerate(self.pendingRequests):
if req['priority'] > priority:
self.pendingRequests.insert(i,req)
break
else:
# didn't find one, tack on the end
self.pendingRequests.append(dictRequest)
# note that we explicitly and purposely do not commit the transaction here.
# the caller is likely to be in a module/collection publish transaction.
# when the caller's transaction commits, the QueueTool change (i.e. adding
# a request) will also commit. thus, the request can not be processed until
# after the publish transaction has completed (i.e. no race condition).
finally:
mutex.release()
security.declarePrivate('find')
def find(self, strKey, listRequests):
"""
Find key in persistent list, looking for duplicates. Return index if found.
"""
for i in range(0,len(listRequests)):
if listRequests[i]['key'] == strKey:
return i
return None
security.declarePrivate('gotRequest')
def gotRequest(self):
"""
Determine if any requests are ready for processing. Return key of first one, if any, else False.
Does dependency checking
"""
# requests must be pending ...
if len(self.pendingRequests) == 0:
return False
# our host server can not be maxed out ...
hostName = getfqdn()
listHostProcess = [entry for entry in self.processingRequests if 'hostName' in entry and entry['hostName'] == hostName]
if listHostProcess is not None:
iProcessMax = hostName in self.dictServers and self.dictServers[hostName] or 1
if len(listHostProcess) >= iProcessMax:
return False
portal = self.portal_url.getPortalObject()
# check requests for dependencies, bail out at first success
for dictRequest in self.pendingRequests:
key = dictRequest['key']
unmetDepends = self.checkDepends(dictRequest)
if unmetDepends:
if unmetDepends[2] == 'reenqueue':
iListIndex = self.find(key, self.pendingRequests)
portal.plone_log("... reenqueue request for key '%s'at index '%s' ..." % (dictRequest['key'],iListIndex))
if iListIndex is not None:
portal.plone_log("... QueueTool.clockTick() has skipped request for key '%s' for failed dependency '%s' ..." % (dictRequest['key'],str(unmetDepends)))
elif unmetDepends[2] == 'fail':
self._email_depend_fail(dictRequest,unmetDepends) # removes req from pending as well
else:
portal.plone_log("... unknown failed dependency action for key '%s' ... skipping." % (dictRequest['key'],))
else:
return key
# fell through loop: no requests w/o unmet dependencies
return False
security.declarePrivate('getRequest')
def getRequest(self, key=None):
"""
Get a request from the pending queue and put it in the processing queue or
get an request already in the processing queue via key.
"""
# caller must acquire mutex lock. caller is responsible for commiting the transaction.
if key is None:
# get a request from the pending queue and put it in the processing queue
reqKey = self.gotRequest()
if reqKey:
iListIndex = self.find(reqKey, self.pendingRequests)
if iListIndex is not None:
dictRequest = self.pendingRequests[iListIndex]
del self.pendingRequests[iListIndex]
self.processingRequests.append(dictRequest)
return dictRequest
else:
return None
else:
return None
else:
# get an request already in the processing queue
iListIndex = self.find(key, self.processingRequests)
if iListIndex is not None:
dictRequest = self.processingRequests[iListIndex]
return dictRequest
else:
return None
security.declarePrivate('start')
def start(self, key):
"""
Start processing the queued request.
"""
# start() acquires mutex lock and is responsible for commiting the transaction.
return
security.declarePrivate('stop')
def stop(self, key):
"""
Stop processing the queued request.
"""
# stop() acquires mutex lock and is responsible for commiting the transaction.
mutex.acquire()
try:
self.removeRequest(key)
# should only commit the above change
transaction.commit()
#import pdb; pdb.set_trace()
#savepoint = transaction.savepoint()
#for i in range(0,3):
# try:
# self.removeRequest(key)
# # should only commit the above change
# transaction.commit()
# except Exception, e:
# print "failure:", e
# self.plone_log("... QueueTool.stop() failed to commit removing the reuest from the processing queue ... %s ..." % str(e))
# savepoint.rollback()
# pass
finally:
mutex.release()
security.declarePrivate('removeRequest')
def removeRequest(self, key, fromList=None):
"""
Remove a request from the processing queue.
"""
# caller must acquire mutex lock. caller is responsible for commiting the transaction.
if not fromList:
fromList = self.processingRequests
iListIndex = self.find(key, fromList)
if iListIndex is not None:
del fromList[iListIndex]
security.declarePrivate('cleanupProcessing')
def cleanupProcessing(self):
"""
Cleanup the processingRequests queue. Assume that processing entries that
do not complete in a defined time window cored and did not call removeRequest().
"""
# cleanupProcessing() acquires mutex lock and is responsible for commiting the transaction.
mutex.acquire()
try:
timeNow = datetime.now()
bChanged = False
for request in self.processingRequests:
timeProcessStarted = 'timeRequestProcessed' in request and request['timeRequestProcessed'] or None
childPid = request.get('pid')
reqHostName = request.get('hostName')
myHostName = getfqdn()
if childPid and reqHostName == myHostName:
try:
os.kill(childPid, 0)
except OSError:
if timeProcessStarted is not None:
self._email_cleanup_message(request,0)
bChanged = True
if timeProcessStarted is not None:
timeProcessing = timeNow - timeProcessStarted
timeProcessingMax = timedelta(seconds=self._getProcessingMax(request))
if timeProcessing > timeProcessingMax:
self._email_cleanup_message(request,timeProcessingMax)
bChanged = True
if bChanged:
# commit the queue change(s)
transaction.commit()
finally:
mutex.release()
security.declarePrivate('_get_ProcessingMax')
def _getProcessingMax(self,request):
# Get process max timeout, checking for exact queue key, keyclass, and default, in that order.
key = request['key']
key_class = key.split('_')[0]
windows = self.secProcessWindows
return windows.get(key, windows.get(key_class, self.secDefaultProcessWindow))
security.declarePrivate('_email_cleanup_message')
def _email_cleanup_message(self,request,timeout):
# email techsupport to notify that request has been forcibly been removed from the queue
mailhost = self.MailHost
portal = self.portal_url.getPortalObject()
mfrom = portal.getProperty('email_from_address')
mto = portal.getProperty('techsupport_email_address') or mfrom
if mto:
subject = "Request '%s' was forcibly removed from processing queue." % request['key']
messageText = "The request did not successfully complete within the required time window of %s. The request may have crashed, may be in an infinite loop, or may just be taking a really long time.\n\nThe server and pid referenced in the request need to be checked to see if the process is still active. If still active, the process may need to be killed. The QueueTool will no longer consider this an active request.\n\nThe complete request was:\n\n%s\n\n" % (timeout,str(request))
self._mailhost_send(messageText, mto, mfrom, subject)
self.removeRequest(request['key'])
self.processingRequests._p_changed = 1
security.declarePrivate('_mailhost_send')
def _mailhost_send(self, messageText, mto, mfrom, subject):
"""attempt to send mail: log if fail"""
portal = self.portal_url.getPortalObject()
try:
mailhost.send(messageText, mto, mfrom, subject)
except:
portal.plone_log("Can't send email:\n\nSubject:%s\n\n%s" % (subject,messageText))
security.declarePublic('ping')
def ping(self):
"""Test method for ClockServer. Logs when called."""
self.plone_log("... QueueTool.ping() has been called ...")
security.declarePublic('clockTick')
def clockTick(self):
"""
Called on a clock event every N seconds. If work is available, a child zopectl process
is launched.
"""
# clockTick() acquires mutex lock and is responsible for commiting the transaction.
mutex.acquire()
portal = self.portal_url.getPortalObject()
try:
if self.gotRequest():
dictRequest = self.getRequest()
if dictRequest:
transaction.commit() # make list change visible to child instance - other way was race condition
self.launchRequest(dictRequest)
portal.plone_log("... QueueTool.clockTick() has spawned a child process for key '%s' ..." % dictRequest['key'])
finally:
mutex.release()
# removed "expired" request in processing
self.cleanupProcessing()
security.declarePublic('checkDepends')
def checkDepends(self, request):
"""Check the request dependecies, in declared order. Return first unmet
one. On success of all (or no declared dependencies) return 'None'"""
for dep in request.get('depends',[]):
try:
checker = getattr(self,"%s_dep" % dep[1])
if checker(request,dep):
return dep
except AttributeError:
self.portal_url.getPortalObject().plone_log('Skipping badly declared dependency: %s' % (dep,))
pass
return None
security.declarePrivate('_email_depend_fail')
def _email_depend_fail(self,request,depend):
# email techsupport to notify that request has been forcibly been removed from the queue
mailhost = self.MailHost
portal = self.portal_url.getPortalObject()
mfrom = portal.getProperty('email_from_address')
mto = portal.getProperty('techsupport_email_address') or mfrom
if mto:
subject = "Request '%s' was removed from pending queue. (dependency)" % request['key']
messageText = "The request declared a dependency that was not available at the time of processing. The QueueTool will no longer consider this an active request.\n\nThe failed dependency was:\n\n%s\n\nThe complete request was:\n\n%s\n\n" % (str(depend),str(request))
self._mailhost_send(messageText, mto, mfrom, subject)
self.removeRequest(request['key'], self.pendingRequests)
self.pendingRequests._p_changed = 1
security.declarePrivate('launchRequest')
def launchRequest(self, dictRequest):
""" Launch the input request.
"""
# caller must acquire mutex lock. caller is responsible for commiting the transaction.
portal = self.portal_url.getPortalObject()
if dictRequest is not None:
key = dictRequest['key'] or None
iListIndex = key is not None and self.find(key, self.processingRequests)
if iListIndex is not None:
# request is in the processing queue
if 'timeRequestProcessed' not in dictRequest:
# request has not already been started
dictRequest['timeRequestProcessed'] = datetime.now()
# dictRequest['ip'] = getipaddr()
dictRequest['hostName'] = getfqdn()
# dictRequest['pid'] = -1
key = dictRequest['key']
serverURL = dictRequest.get('serverURL',self.REQUEST['SERVER_URL'])
callbackrequesthandler = dictRequest['requestHandler']
cmd = os.path.join(INSTANCE_HOME, 'bin', 'zopectl')
callback = os.path.join(INSTANCE_HOME, 'Products', callbackrequesthandler)
#pid = os.spawnl(os.P_NOWAIT, cmd, 'zopectl', 'run', callback, key)
# call newer, better python API which allows for the stdout and stderr to be captured
import subprocess
f1 = open('/tmp/%s.stdout' % key, 'w')
f2 = open('/tmp/%s.stderr' % key, 'w')
pid = subprocess.Popen([cmd, "run", callback, key, serverURL], stdout=f1, stderr=f2).pid
# child will call start() and stop()
dictRequest['pid'] = pid
self.processingRequests._p_changed = 1
security.declareProtected(ManagePermission, 'manage_tick')
def manage_tick(self):
"""ZMI-useful variant of 'clockTick'. Goes back to overview page of tool.
From anywhere else, use 'clockTick' instead.
"""
self.clockTick()
self.REQUEST.RESPONSE.redirect('manage_overview')
return
security.declareProtected(ManagePermission, 'manage_cut_in_line')
def manage_cut_in_line(self, key=""):
"""Move request to the head of the pendingRequests queue.
"""
# manage_cut_in_line() acquires mutex lock and is responsible for commiting the transaction.
mutex.acquire()
try:
portal = self.portal_url.getPortalObject()
iListIndex = self.find(key, self.pendingRequests)
if iListIndex is not None and iListIndex > 0:
dictRequest = self.pendingRequests[iListIndex] or None
if dictRequest is not None:
del self.pendingRequests[iListIndex]
self.pendingRequests.insert(0, dictRequest)
transaction.commit()
portal.plone_log("... QueueTool.manage_cut_in_line: '%s' is cutting in line ..." % key)
self.REQUEST.RESPONSE.redirect('manage_overview')
finally:
mutex.release()
return
security.declareProtected(ManagePermission, 'manage_remove')
def manage_remove(self, key="", queue="pending"):
"""Remove request from a queue list
"""
# manage_cut_in_line() acquires mutex lock and is responsible for commiting the transaction.
mutex.acquire()
if queue == "pending":
fromList = self.pendingRequests
elif queue == "processing":
fromList = self.processingRequests
else:
return
try:
self.removeRequest(key,fromList)
portal = self.portal_url.getPortalObject()
portal.plone_log("... QueueTool.manage_remove: '%s' has been removed ..." % key)
self.REQUEST.RESPONSE.redirect('manage_overview')
finally:
mutex.release()
return
security.declareProtected(ManagePermission, 'getProcessingRequests')
def getProcessingRequests(self):
"""Get process requests."""
return self.processingRequests
security.declareProtected(ManagePermission, 'getPendingRequests')
def getPendingRequests(self):
"""Get pending requests."""
return self.pendingRequests
security.declarePublic('pending_count')
def pending_count(self):
"""Return number of pending entries in the queue. Useful for monitoring."""
return len(self.getPendingRequests())
security.declarePublic('processing_count')
def processing_count(self):
"""Return number of pending entries in the queue. Useful for monitoring."""
return len(self.getProcessingRequests())
def checkTheWorld(self):
# Test constructors
u = PersistentList()
u0 = PersistentList(l0)
u1 = PersistentList(l1)
u2 = PersistentList(l2)
uu = PersistentList(u)
uu0 = PersistentList(u0)
uu1 = PersistentList(u1)
uu2 = PersistentList(u2)
v = PersistentList(tuple(u))
class OtherList:
def __init__(self, initlist):
self.__data = initlist
def __len__(self):
return len(self.__data)
def __getitem__(self, i):
return self.__data[i]
v0 = PersistentList(OtherList(u0))
vv = PersistentList("this is also a sequence")
# Test __repr__
eq = self.assertEqual
eq(str(u0), str(l0), "str(u0) == str(l0)")
eq(repr(u1), repr(l1), "repr(u1) == repr(l1)")
eq(`u2`, `l2`, "`u2` == `l2`")
# Test __cmp__ and __len__
def mycmp(a, b):
r = cmp(a, b)
if r < 0: return -1
if r > 0: return 1
return r
all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2]
for a in all:
for b in all:
eq(mycmp(a, b), mycmp(len(a), len(b)),
"mycmp(a, b) == mycmp(len(a), len(b))")
# Test __getitem__
for i in range(len(u2)):
eq(u2[i], i, "u2[i] == i")
# Test __setitem__
uu2[0] = 0
uu2[1] = 100
try:
uu2[2] = 200
except IndexError:
pass
else:
raise TestFailed("uu2[2] shouldn't be assignable")
# Test __delitem__
del uu2[1]
del uu2[0]
try:
del uu2[0]
except IndexError:
pass
else:
raise TestFailed("uu2[0] shouldn't be deletable")
# Test __getslice__
for i in range(-3, 4):
eq(u2[:i], l2[:i], "u2[:i] == l2[:i]")
eq(u2[i:], l2[i:], "u2[i:] == l2[i:]")
for j in range(-3, 4):
eq(u2[i:j], l2[i:j], "u2[i:j] == l2[i:j]")
# Test __setslice__
for i in range(-3, 4):
u2[:i] = l2[:i]
eq(u2, l2, "u2 == l2")
u2[i:] = l2[i:]
eq(u2, l2, "u2 == l2")
for j in range(-3, 4):
u2[i:j] = l2[i:j]
eq(u2, l2, "u2 == l2")
uu2 = u2[:]
uu2[:0] = [-2, -1]
eq(uu2, [-2, -1, 0, 1], "uu2 == [-2, -1, 0, 1]")
uu2[0:] = []
eq(uu2, [], "uu2 == []")
# Test __contains__
for i in u2:
self.failUnless(i in u2, "i in u2")
for i in min(u2)-1, max(u2)+1:
self.failUnless(i not in u2, "i not in u2")
# Test __delslice__
uu2 = u2[:]
del uu2[1:2]
del uu2[0:1]
eq(uu2, [], "uu2 == []")
uu2 = u2[:]
del uu2[1:]
del uu2[:1]
eq(uu2, [], "uu2 == []")
# Test __add__, __radd__, __mul__ and __rmul__
#self.failUnless(u1 + [] == [] + u1 == u1, "u1 + [] == [] + u1 == u1")
self.failUnless(u1 + [1] == u2, "u1 + [1] == u2")
#self.failUnless([-1] + u1 == [-1, 0], "[-1] + u1 == [-1, 0]")
self.failUnless(u2 == u2*1 == 1*u2, "u2 == u2*1 == 1*u2")
self.failUnless(u2+u2 == u2*2 == 2*u2, "u2+u2 == u2*2 == 2*u2")
self.failUnless(u2+u2+u2 == u2*3 == 3*u2, "u2+u2+u2 == u2*3 == 3*u2")
# Test append
u = u1[:]
u.append(1)
eq(u, u2, "u == u2")
# Test insert
u = u2[:]
u.insert(0, -1)
eq(u, [-1, 0, 1], "u == [-1, 0, 1]")
# Test pop
u = PersistentList([0, -1, 1])
u.pop()
eq(u, [0, -1], "u == [0, -1]")
u.pop(0)
eq(u, [-1], "u == [-1]")
# Test remove
u = u2[:]
u.remove(1)
eq(u, u1, "u == u1")
# Test count
u = u2*3
eq(u.count(0), 3, "u.count(0) == 3")
eq(u.count(1), 3, "u.count(1) == 3")
eq(u.count(2), 0, "u.count(2) == 0")
# Test index
eq(u2.index(0), 0, "u2.index(0) == 0")
eq(u2.index(1), 1, "u2.index(1) == 1")
try:
u2.index(2)
except ValueError:
pass
else:
raise TestFailed("expected ValueError")
# Test reverse
u = u2[:]
u.reverse()
eq(u, [1, 0], "u == [1, 0]")
u.reverse()
eq(u, u2, "u == u2")
# Test sort
u = PersistentList([1, 0])
u.sort()
eq(u, u2, "u == u2")
# Test extend
u = u1[:]
u.extend(u2)
eq(u, u1 + u2, "u == u1 + u2")
def test_conflict_errors(self):
"""
Here we realize that conflict errors occur only occur when concurrent modifications
on a particular container (with specific oid) occur concurrently. Updates can still
be lost if a branch of the object tree is disconnected from the root while one of
its leaves gets updated.
Similarly, readCurrent() only protects a specific container of the object tree,
which can still be disconnected from the root by a transaction, while its content
is updated by another transaction.
"""
conn = self.conn
root = conn.root
root.stuff = PersistentList([9])
root.origin = PersistentList([3])
root.target = PersistentList([8])
root.dummy1 = PersistentList([9])
transaction.commit()
# basic conflict on root #
pool.apply(delete_container, args=(self.db, "dummy1"))
root.dummy2 = 5
self.assertRaises(ConflictError, transaction.commit) # conflict !!
self.assertRaises(TransactionFailedError, transaction.commit) # transaction broken
transaction.abort()
self.assertFalse(hasattr(root, "dummy2")) # rolled back
# no conflict when a branch gets detached while leaf is updated
container = root.stuff
pool.apply(delete_container, args=(self.db, "stuff"))
container[0] = 88
transaction.commit()
self.assertFalse(hasattr(root, "stuff")) # update lost
# without readCurrent() - lost update #
root.origin = PersistentList([13])
value = root.origin
pool.apply(transfer_list_value, args=(self.db, "origin", "target"))
root.target = value
transaction.commit()
self.assertEqual(root.target, PersistentList([13])) # we lost [3]
# with readCurrent() and container update - ReadConflictError raised! #
root.origin = PersistentList([17])
transaction.commit()
res = conn.readCurrent(root.target) # container object selected !!
assert res is None # no return value expected
value = root.target
pool.apply(transfer_list_value, args=(self.db, "origin", "target"))
root.othertarget = value
self.assertRaises(Exception, transaction.commit)
self.assertEqual(root.target, PersistentList([17])) # auto refreshing occurred
self.assertFalse(hasattr(root, "othertarget")) # auto refreshing occurred
self.assertRaises(Exception, transaction.commit) # but transaction still broken
transaction.abort()
transaction.commit() # now all is ok once again
# with readCurrent() and container deletion - somehow lost update! #
value = root.origin[0]
res = conn.readCurrent(root.origin) # container object selected !!
assert res is None # no return value expected
pool.apply(delete_container, args=(self.db, "origin"))
root.target[0] = value # we use a value whose origin has now been deleted in other thread
transaction.commit() # here it's OK, the deleted object still remains in the DB history even if unreachable
def __init__(self, folder_dir):
self._dir = folder_dir
self.classifier = PBayes()
self.hams = PersistentList()
self.spams = PersistentList()
class Dispatcher:
""" Sends commands to and receives results from a web browser.
Dispatcher:
_Computer Science_: A routine that controls the order in which input
and output devices obtain access to the processing system.
(source: http://dictionary.reference.com/search?q=dispatcher)
In Selenium, the Dispatcher class takes commands to be executed and puts
them in a queue for a web browser to read.
When the browser reads this queue for a command to execute, it adds the
previous command's result to the Dispatcher's result queue.
"""
def __init__(self):
self.queue_timeout = QUEUE_TIMEOUT
self._commands = PersistentList()
self._results = PersistentList()
#
# Command queue methods
#
def clearCommands(self):
del self._commands[:]
return 'Commands cleared'
def addCommand(self, command, REQUEST=None):
""" Add a command to the commands queue """
self._commands.append(command)
# super important to commit the change at this point...
# In some methods (like webDriver and apiDriver), there is a later call
# to '_p_jar.sync(), and that
# call would 'roll-back' this action if we didn't commit here.
get_transaction().commit()
return 'Command added'
def getCommand(self, REQUEST=None):
""" Retreive a command from the commands queue """
if len(self._commands) != 0:
response = self._commands.pop(0)
else:
# if the queue is empty wait to see if any commands are entered
# until we're forced to time out the request.
elapsed_time = 0
step = .5 # in seconds
while elapsed_time <= self.queue_timeout:
time.sleep(step) #in seconds
elapsed_time += step
# Syncronize with the ZODB in case of any recent updates.
# If this is being run as a unit test, we won't have an
# attribute named "_p_jar" that we can sync()
if not getattr(self,'_p_jar', None):
self._p_jar.sync()
try:
response = self._commands.pop(0)
break
except IndexError:
response = 'ERROR: Command queue was empty'
return response
def getCommandQueueSize(self, REQUEST=None):
""" Query the size of the commands queue """
size = len(self._commands)
return size
#
# Result queue methods
#
def clearResults(self):
del self._results[:]
return 'Results cleared'
def addResult(self, result, REQUEST=None):
""" Add a result to the results queue """
self._results.append(result)
# super important to commit the change at this point...
# In some methods (like webDriver and apiDriver), there is a later call
# to '_p_jar.sync(), and that
# call would 'roll-back' this action if we didn't commit here.
get_transaction().commit()
return 'Result added'
def getResult(self, REQUEST=None):
""" Retrieve a result from the results queue """
if len(self._results) != 0:
response = self._results.pop(0)
else:
# if the queue is empty wait to see if any results are entered
# until we're forced to time out the request.
elapsed_time = 0
step = .5 # in seconds
while 1:
time.sleep(step) #in seconds
elapsed_time += step
# Syncronize with the ZODB in case of any recent updates.
# If this is being run as a unit test, we won't have an
# attribute named "_p_jar" that we can sync()
if not getattr(self,'_p_jar', None):
self._p_jar.sync()
try:
response = self._results.pop(0)
break
except IndexError:
if elapsed_time >= self.queue_timeout:
response = 'ERROR: Result queue was empty'
break
return response
def getResultQueueSize(self, REQUEST=None):
""" Query the size of the results queue """
size = len(self._results)
return size
#
# Misc. methods
#
def setTimeout(self,timeout):
""" Set the timeout period in seconds that a browser or driver should
wait before timing out"""
self.queue_timeout = timeout
return "Timeout set to %s seconds" % timeout
def webDriver(self, REQUEST=None):
""" Gets a command from the command queue. Also, adds a result
to the result queue, unless the seleniumStart form paramter
(seleniumStart=true) is present.
Note: this method is usually called from a web browser
as "http://<server>/selenium-driver/driver"
"""
if REQUEST:
command_result = REQUEST.form.get('commandResult')
selenium_start = REQUEST.form.get('seleniumStart')
# If 'seleniumStart' is a parameter on the request,
# it means this is the first time hitting the driver,
# and therefore, there is no previous result to post to
# the results queue.
if selenium_start:
self.clearResults()
if command_result:
self.addResult(command_result)
return self.getCommand()
else:
return "ERROR: Missing an HTTP REQUEST"
def apiDriver(self, command_string="", REQUEST=None):
""" Adds a command to the command queue, and gets a result
from the result queue.
Note: this method is usually called from Python
source code, not a web browser."""
if REQUEST:
command_string = REQUEST.form.get('command_string')
self.addCommand(command_string)
# if test is complete, don't try retrieving a response from the browser.
if command_string.find('testComplete') >= 0:
return 'test complete'
else:
return self.getResult()
def __init__(self, parent):
Base.__init__(self, parent)
self.__pers_list = PersistentList()
def setCriteriaForType(self, bib_type, criteria):
"""update criteria for a bibliography type given"""
self.duplicates_criteria[bib_type] = PersistentList(criteria)
#FIXME may need to check if any change has been done
self.criteriaUpdated = True
return True
def ___test_huge_db_ghosting_system():
"""
Interactive testcase, to demonstrate the behaviour of ZODB regarding memory management and ghost objects.
Launch it with a "top"-like window opened next to it.
MIGHT TRIGGER THIS WARNING:
<...>Connection.py:550: UserWarning: The <class 'persistent.list.PersistentList'>
object you're saving is large. (20001339 bytes.)
Perhaps you're storing media which should be stored in blobs.
Perhaps you're using a non-scalable data structure, such as a
PersistentMapping or PersistentList.
Perhaps you're storing data in objects that aren't persistent at
all. In cases like that, the data is stored in the record of the
containing persistent object.
In any case, storing records this big is probably a bad idea.
If you insist and want to get rid of this warning, use the
large_record_size option of the ZODB.DB constructor (or the
large-record-size option in a configuration file) to specify a larger
size.
Playing with this test shows that:
- contants of persistent lists and mappings are really only loaded when accessed (eg. when lookup is done on them..)
- non persistent types (list(), dict()) are badly treated, and remain in memory even when committed to file
"""
use_non_persistent_types = False
PersistentList = globals()["PersistentList"]
PersistentMapping = globals()["PersistentMapping"]
if use_non_persistent_types:
PersistentList = list
PersistentMapping = dict
db, connection, root = _get_huge_db_root()
root["data"] = PersistentList(PersistentMapping({"toto": "tata"*random.randint(5000, 6000)}) for i in range(20000))
root["data"][0] = PersistentMapping({"toto": "tata"*5000000}) # HUGE first element
print("We have a HUGE database filled with transient data now!")
time.sleep(5)
transaction.commit()
print("We have committed the transaction!")
time.sleep(5)
connection.close()
db.close()
# ---------
db, connection, root = _get_huge_db_root()
print("We have reopened the huge database now!")
time.sleep(5)
data = root["data"]
print("We have accessed data list!")
time.sleep(5)
var1 = data[0]
print("We have accessed first element of data list!")
time.sleep(5)
var1 = data[0]["toto"]
print("We have unghosted first element of data list!")
time.sleep(5)
for i in data:
i # no unghosting
print("We have accessed all elements of data list!")
time.sleep(5)
for i in data:
i["toto"] # THIS removes the ghosting effect on element
print("We have unghosted all elements of data list!")
time.sleep(15)
def __init__(self):
self._libraries = PersistentList()
self._res_types = PersistentMapping()
self.linkbyuid = False
def __init__(self, folder_dir):
self._dir = folder_dir
self.classifier = PBayes()
self.hams = PersistentList()
self.spams = PersistentList()
class CardStack(Persistent):
def __init__(self):
self._v_current_card = 0
self._v_text = None
self._v_cardNumber = None
self.last_id = 0
self.card_list = PersistentList()
self.to_be_indexed = PersistentList()
self.card_store = OOBTree.OOBTree()
self.hint_store = OOBTree.OOBTree()
self.CreateCard(at_end=1, and_set=0)
def SetHints(self, id, hints):
if self.to_be_indexed[0] == id:
self.hint_store[id] = hints
del self.to_be_indexed[0]
else:
print "Not saving hints for card id", id, "because card has been deleted."
def StoreText(self, id, text):
self.card_store[id] = self.Compress(text)
self.hint_store[id] = None
if id not in self.to_be_indexed:
self.to_be_indexed.append(id)
get_transaction().commit()
def GetText(self, id):
return self.Decompress(self.card_store[id])
def DeleteCard(self):
if len(self.card_list) <= 1: return
id = self.card_list[self._v_current_card]
del self.card_list[self._v_current_card]
del self.card_store[id]
del self.hint_store[id]
self.SetCard(self._v_current_card)
def NewCard(self):
self.SaveCurrentCard()
self.CreateCard()
def CreateCard(self, at_end=0, text="", and_set=1):
id = self.last_id + 1
self.last_id = id
if at_end:
self.card_list.append(id)
else:
self.card_list.insert(self._v_current_card + 1, id)
self.StoreText(id, text)
if and_set:
self.SetCard(self._v_current_card + 1)
def SetCard(self, card_num):
if card_num >= len(self.card_list):
card_num = len(self.card_list) - 1
if card_num < 0: card_num = 0
self._v_current_card = card_num
id = self.card_list[card_num]
card_text = self.GetText(id)
self._v_text.text = card_text
self._v_cardNumber.text = 'Card: ' + str(card_num + 1)
print "Card", card_num + 1, "of", len(self.card_list)
def SaveCurrentCard(self):
text = self._v_text.text
id = self.card_list[self._v_current_card]
self.StoreText(id, text)
def Decompress(self, text):
if len(text) == 0: return text
flag_index = len(text) - 1
flag = text[flag_index]
if flag == chr(0): return text[0:-1]
from zlib import decompress
return decompress(text[0:-1])
def Compress(self, text):
old_len = len(text)
if old_len == 0: return text
from zlib import compress
start_time = time.time()
compressed = compress(text, 9)
elapsed_time = time.time() - start_time
new_len = len(compressed)
if new_len < old_len:
print "compressed", old_len, "bytes by", old_len - new_len, "bytes"
return compressed + chr(1)
print "leaving text of len", old_len, "uncompressed."
return text + chr(0)
def GoNext(self, dummy=""):
self.SaveCurrentCard()
self.SetCard(self._v_current_card + 1)
def GoPrev(self, dummy=""):
self.SaveCurrentCard()
self.SetCard(self._v_current_card - 1)
def GoFirst(self):
self.SaveCurrentCard()
self.SetCard(0)
def GoLast(self):
self.SaveCurrentCard()
self.SetCard(len(self.card_list) - 1)
def ImportStack(self):
"""imports stacks of the format described in
Managing Gigabytes
Compressing and Indexing Documents and Images
Second Edition, 1999
http://www.cs.mu.oz.au/mg/
source at http://www.cs.mu.oz.au/mg/mg-1.2.1.tar.gz
"""
wildcard = "stack files (*.stack)|*.stack|All Files (*.*)|*.*"
result = dialog.openFileDialog(None, "Import which stack?", '', '',
wildcard)
if result.accepted:
path = result.paths[0]
file = open(path, "r")
cards = file.read().split(str(chr(2)))
for text in cards:
self.CreateCard(at_end=1,
text=re.sub("\r", "\n", text),
and_set=0)
file.close()
self.SetCard(0)
def __init__(self):
self.queue_timeout = QUEUE_TIMEOUT
self._commands = PersistentList()
self._results = PersistentList()
