class List(Base): def __init__(self, parent): Base.__init__(self, parent) self.__pers_list = PersistentList() def show(self): for x in self: print x def __repr__(self): return self.__pers_list.__repr__() def __getitem__(self, key): return self.__pers_list.__getitem__(key) def __len__(self): return self.__pers_list.__len__() def __getattr__(self, name): if name in [ "append", "extend", "insert", "pop", "remove", "reverse", "sort" ]: return getattr(self.__pers_list, name) return Base.__getattr__(self, name)
class Profile(Persistent): FolderClass = Folder def __init__(self, folder_dir): self._dir = folder_dir self.classifier = PBayes() self.hams = PersistentList() self.spams = PersistentList() def add_ham(self, folder): p = os.path.join(self._dir, folder) f = self.FolderClass(p) self.hams.append(f) def add_spam(self, folder): p = os.path.join(self._dir, folder) f = self.FolderClass(p) self.spams.append(f) def update(self): """Update classifier from current folder contents.""" changed1 = self._update(self.hams, False) changed2 = self._update(self.spams, True) ## if changed1 or changed2: ## self.classifier.update_probabilities() get_transaction().commit() log("updated probabilities") def _update(self, folders, is_spam): changed = False for f in folders: log("update from %s" % f.path) added, removed = f.read() if added: log("added %d" % len(added)) if removed: log("removed %d" % len(removed)) get_transaction().commit() if not (added or removed): continue changed = True # It's important not to commit a transaction until # after update_probabilities is called in update(). # Otherwise some new entries will cause scoring to fail. for msg in added.keys(): self.classifier.learn(tokenize(msg), is_spam) del added get_transaction().commit(1) log("learned") for msg in removed.keys(): self.classifier.unlearn(tokenize(msg), is_spam) if removed: log("unlearned") del removed get_transaction().commit(1) return changed
def __init__(self): self._v_current_card = 0 self._v_text = None self._v_cardNumber = None self.last_id = 0 self.card_list = PersistentList() self.to_be_indexed = PersistentList() self.card_store = OOBTree.OOBTree() self.hint_store = OOBTree.OOBTree() self.CreateCard(at_end=1, and_set=0)
def __init__(self): """Initialize (singleton!) tool object.""" # user configurable self.dictServers = {} self.secProcessWindows = {} self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW # unknown to the user self.pendingRequests = PersistentList([]) self.processingRequests = PersistentList([]) log('__init__ completed.') # this no workeee???
def update_dataflow_uris(root, commit=False): candidates = filter_objects(root, filter_level=0) counter = 0 changes_log.info('DATAFLOW URIS UPDATES') transaction.savepoint() for obj in candidates: dataflow_uris = getattr(obj, 'dataflow_uris', None) dataflow_uri = '' corrected_uris = PersistentList() corrected_uri = '' message = '' message += ('{type:21}: {url}'.format( type=(obj.meta_type[:17]+'... ' if len(obj.meta_type)>20 else obj.meta_type), url=obj.absolute_url_path())) for uri in getattr(obj, 'dataflow_uris', []): corrected_uris.append(uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu')) if not(getattr(obj, 'dataflow_uris', None) is None): obj.dataflow_uris = corrected_uris if not dataflow_uris == corrected_uris: message += '\n{attr:21}: {before} -> {after}'.format( attr='dataflow_uris', before=dataflow_uris, after=obj.dataflow_uris) if getattr(obj, 'dataflow_uri', None): dataflow_uri = obj.dataflow_uri corrected_uri = dataflow_uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu') obj.dataflow_uri = corrected_uri if not dataflow_uri == corrected_uri: message += '\n{attr:21}: {before} -> {after}'.format( attr='dataflow_uri', before=dataflow_uri, after=corrected_uri) if getattr(obj, 'country', None): country_uri = obj.country corrected_country = country_uri.replace('rod.eionet.eu.int', 'rod.eionet.europa.eu') obj.country = corrected_country if not country_uri == corrected_country: message += '\n{attr:21}: {before} -> {after}'.format( attr='country', before=country_uri, after=corrected_country) if not obj._p_changed: message+="\nno changes made" message+="\n" changes_log.info(message) counter+=1 if counter % 1000 == 0: transaction.savepoint() if commit: transaction.commit() changes_log.info('ALL CHANGES COMMITED TO ZODB!')
def testNewFolderCriterias(self): """ testes that the import_criterias for a fold changed after being modifieded (folder bg) """ duplicates_criteria = { 'BookReference': PersistentList(('authors', 'title')) } self.second_bf = self.getEmptyBibFolder('second_bib_folder') old_criterias = self.bf.getCriterias() sec_old_criterias = self.second_bf.getCriterias() for bib_type in duplicates_criteria.keys(): self.bf.setCriteriasForType(bib_type, duplicates_criteria[bib_type]) new_criterias = self.bf.getCriterias() sec_new_criterias = self.second_bf.getCriterias() self.assertNotEquals(old_criterias, new_criterias, 'Criterias have not been changed') self.assertEquals(sec_old_criterias, sec_new_criterias, 'Criterias have been changed')
class Profile(Persistent): FolderClass = Folder def __init__(self, folder_dir): self._dir = folder_dir self.classifier = PBayes() self.hams = PersistentList() self.spams = PersistentList() def add_ham(self, folder): p = os.path.join(self._dir, folder) f = self.FolderClass(p) self.hams.append(f) def add_spam(self, folder): p = os.path.join(self._dir, folder) f = self.FolderClass(p) self.spams.append(f) def update(self): """Update classifier from current folder contents.""" changed1 = self._update(self.hams, False) changed2 = self._update(self.spams, True) get_transaction().commit() log("updated probabilities") def _update(self, folders, is_spam): changed = False for f in folders: log("update from %s" % f.path) added, removed = f.read() if added: log("added %d" % len(added)) if removed: log("removed %d" % len(removed)) get_transaction().commit() if not (added or removed): continue changed = True for msg in added.keys(): self.classifier.learn(tokenize(msg), is_spam) del added get_transaction().commit(1) log("learned") for msg in removed.keys(): self.classifier.unlearn(tokenize(msg), is_spam) if removed: log("unlearned") del removed get_transaction().commit(1) return changed
def make_config_persistent(kwargs): """ iterates on the given dictionnary and replace list by persistent list, dictionary by persistent mapping. """ for key, value in kwargs.items(): if type(value) == type({}): p_value = PersistentMapping(value) kwargs[key] = p_value elif type(value) in (type(()), type([])): p_value = PersistentList(value) kwargs[key] = p_value
def test_persistent_types_buglets(self): l = PersistentList([1, 2, 3]) self.assertTrue(isinstance(l, PersistentList)) self.assertFalse(isinstance(l, list)) # dangerous self.assertTrue(isinstance(l[:], PersistentList)) d = PersistentMapping({1: 2}) self.assertTrue(isinstance(d, PersistentMapping)) self.assertFalse(isinstance(d, dict)) # dangerous self.assertTrue(isinstance(d.copy(), PersistentMapping))
def getSelectedCriteria(self, bib_type=None): # migrate CMFBAT v0.8 duplicates engine, mending a linguistic fault-pas if shasattr(self, 'imp_criterias'): print 'CMFBibliographyAT: performing duplicates engine property update - v0.8 -> v0.9 (getSelectedCriteria of %s)' % '/'.join( self.getId()) self.duplicates_criteria = PersistentMapping() self.duplicates_criteria = copy.deepcopy(self.imp_criterias) self.criteriaUpdated = self.criteriasUpdated try: delattr(self, 'imp_criterias') except: pass try: delattr(self, 'criteriasUpdated') except: pass # first call? initialize self.duplicates_criteria bib_tool = getToolByName(self, 'portal_bibliography') if not shasattr(self, 'duplicates_criteria') or not self.duplicates_criteria: if self.getId() == bib_tool.getId(): for reference_type in bib_tool.getReferenceTypes(): self.duplicates_criteria[reference_type] = PersistentList( self._default_duplicates_criteria) self.criteriaUpdated = True else: self.duplicates_criteria = bib_tool.getSelectedCriteria() self.criteriaUpdated = True if not shasattr(self, '_criteria') or not self._criteria: self.initCriteria() # make sure, our selected criteria are in sync with available criteria duplicates_criteria = {} for key in self._criteria.keys(): duplicates_criteria[key] = [ criterion for criterion in self._criteria[key] if self.duplicates_criteria.has_key(key) and ( criterion in self.duplicates_criteria[key]) ] if bib_type: try: duplicates_criteria[bib_type] except KeyError: return False return duplicates_criteria[bib_type] else: return duplicates_criteria
class QueueTool(UniqueObject, SimpleItem): """ Tool for storage of Print files """ id = 'queue_tool' meta_type = 'Queue Tool' manage_options = (({'label':'Overview', 'action':'manage_overview'}, {'label':'Configure', 'action':'manage_configure'} )+ SimpleItem.manage_options ) manage_overview = PageTemplateFile('zpt/manage_overview_queue_tool.zpt', globals()) manage_configure = PageTemplateFile('zpt/manage_configure_queue_tool.zpt', globals()) security = AccessControl.ClassSecurityInfo() # set default time window to 1 hour DEFAULT_PROCESS_TIME_WINDOW = 3600 def __init__(self): """Initialize (singleton!) tool object.""" # user configurable self.dictServers = {} self.secProcessWindows = {} self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW # unknown to the user self.pendingRequests = PersistentList([]) self.processingRequests = PersistentList([]) log('__init__ completed.') # this no workeee??? # Dependency checkers def printtool_dep(self, request, dep): """check for the existence of the given filetype instance (objectId, Version, extenstion) newer than the given timestamp""" pt = getToolByName(self,'rhaptos_print') depkey,deptype,depaction,depdetail = dep ptf_params = (depdetail['objectId'],depdetail['version'],depdetail['extension']) if pt.doesFileExist(*ptf_params): mod_date = DateTime(pt.getModificationDate(*ptf_params)) if mod_date > depdetail['newer']: return None return dep def url_dep(self, request, dep): """check for bits available from the given url, with a given mimetype and newer than the given timestamp""" pass def file_dep(self, request, dep): """check for the existence if a file at the given path, that is newer than the given timestamp""" depkey,deptype,depaction,depdetail = dep try: fstat = os.stat(depdetail['path']) except OSError: return dep if DateTime(fstat.st_ctime) > depdetail['newer']: return None return dep security.declareProtected(ManagePermission, 'manage_queue_tool') def manage_queue_tool(self, dictServers={}, dictProcessWindows={}, secDefaultProcessWindow=DEFAULT_PROCESS_TIME_WINDOW): """ Post creation configuration. See manage_configure_queue_tool.zpt """ self.dictServers = eval(dictServers) # string not dictionary returned self.secProcessWindows = eval(dictProcessWindows) # string not dictionary returned try: self.secDefaultProcessWindow = int(secDefaultProcessWindow) except ValueError: self.secDefaultProcessWindow = self.DEFAULT_PROCESS_TIME_WINDOW security.declareProtected(ManagePermission, 'add') def add(self, key, dictParams, callbackrequesthandler, priority=1): """ Add a request to the Pending Queue. """ # add() acquires mutex lock. caller is responsible for commiting the transaction. mutex.acquire() try: iListIndex = self.find(key, self.pendingRequests) if iListIndex is not None: # remove duplicate del self.pendingRequests[iListIndex] dictRequest = dictParams.copy() dictRequest['key'] = key dictRequest['requestHandler'] = callbackrequesthandler dictRequest['timeRequestMade'] = datetime.now() dictRequest['priority'] = priority # Walk list, insert immediately before first entry that is lower priority (higher value) for i,req in enumerate(self.pendingRequests): if req['priority'] > priority: self.pendingRequests.insert(i,req) break else: # didn't find one, tack on the end self.pendingRequests.append(dictRequest) # note that we explicitly and purposely do not commit the transaction here. # the caller is likely to be in a module/collection publish transaction. # when the caller's transaction commits, the QueueTool change (i.e. adding # a request) will also commit. thus, the request can not be processed until # after the publish transaction has completed (i.e. no race condition). finally: mutex.release() security.declarePrivate('find') def find(self, strKey, listRequests): """ Find key in persistent list, looking for duplicates. Return index if found. """ for i in range(0,len(listRequests)): if listRequests[i]['key'] == strKey: return i return None security.declarePrivate('gotRequest') def gotRequest(self): """ Determine if any requests are ready for processing. Return key of first one, if any, else False. Does dependency checking """ # requests must be pending ... if len(self.pendingRequests) == 0: return False # our host server can not be maxed out ... hostName = getfqdn() listHostProcess = [entry for entry in self.processingRequests if 'hostName' in entry and entry['hostName'] == hostName] if listHostProcess is not None: iProcessMax = hostName in self.dictServers and self.dictServers[hostName] or 1 if len(listHostProcess) >= iProcessMax: return False portal = self.portal_url.getPortalObject() # check requests for dependencies, bail out at first success for dictRequest in self.pendingRequests: key = dictRequest['key'] unmetDepends = self.checkDepends(dictRequest) if unmetDepends: if unmetDepends[2] == 'reenqueue': iListIndex = self.find(key, self.pendingRequests) portal.plone_log("... reenqueue request for key '%s'at index '%s' ..." % (dictRequest['key'],iListIndex)) if iListIndex is not None: portal.plone_log("... QueueTool.clockTick() has skipped request for key '%s' for failed dependency '%s' ..." % (dictRequest['key'],str(unmetDepends))) elif unmetDepends[2] == 'fail': self._email_depend_fail(dictRequest,unmetDepends) # removes req from pending as well else: portal.plone_log("... unknown failed dependency action for key '%s' ... skipping." % (dictRequest['key'],)) else: return key # fell through loop: no requests w/o unmet dependencies return False security.declarePrivate('getRequest') def getRequest(self, key=None): """ Get a request from the pending queue and put it in the processing queue or get an request already in the processing queue via key. """ # caller must acquire mutex lock. caller is responsible for commiting the transaction. if key is None: # get a request from the pending queue and put it in the processing queue reqKey = self.gotRequest() if reqKey: iListIndex = self.find(reqKey, self.pendingRequests) if iListIndex is not None: dictRequest = self.pendingRequests[iListIndex] del self.pendingRequests[iListIndex] self.processingRequests.append(dictRequest) return dictRequest else: return None else: return None else: # get an request already in the processing queue iListIndex = self.find(key, self.processingRequests) if iListIndex is not None: dictRequest = self.processingRequests[iListIndex] return dictRequest else: return None security.declarePrivate('start') def start(self, key): """ Start processing the queued request. """ # start() acquires mutex lock and is responsible for commiting the transaction. return security.declarePrivate('stop') def stop(self, key): """ Stop processing the queued request. """ # stop() acquires mutex lock and is responsible for commiting the transaction. mutex.acquire() try: self.removeRequest(key) # should only commit the above change transaction.commit() #import pdb; pdb.set_trace() #savepoint = transaction.savepoint() #for i in range(0,3): # try: # self.removeRequest(key) # # should only commit the above change # transaction.commit() # except Exception, e: # print "failure:", e # self.plone_log("... QueueTool.stop() failed to commit removing the reuest from the processing queue ... %s ..." % str(e)) # savepoint.rollback() # pass finally: mutex.release() security.declarePrivate('removeRequest') def removeRequest(self, key, fromList=None): """ Remove a request from the processing queue. """ # caller must acquire mutex lock. caller is responsible for commiting the transaction. if not fromList: fromList = self.processingRequests iListIndex = self.find(key, fromList) if iListIndex is not None: del fromList[iListIndex] security.declarePrivate('cleanupProcessing') def cleanupProcessing(self): """ Cleanup the processingRequests queue. Assume that processing entries that do not complete in a defined time window cored and did not call removeRequest(). """ # cleanupProcessing() acquires mutex lock and is responsible for commiting the transaction. mutex.acquire() try: timeNow = datetime.now() bChanged = False for request in self.processingRequests: timeProcessStarted = 'timeRequestProcessed' in request and request['timeRequestProcessed'] or None childPid = request.get('pid') reqHostName = request.get('hostName') myHostName = getfqdn() if childPid and reqHostName == myHostName: try: os.kill(childPid, 0) except OSError: if timeProcessStarted is not None: self._email_cleanup_message(request,0) bChanged = True if timeProcessStarted is not None: timeProcessing = timeNow - timeProcessStarted timeProcessingMax = timedelta(seconds=self._getProcessingMax(request)) if timeProcessing > timeProcessingMax: self._email_cleanup_message(request,timeProcessingMax) bChanged = True if bChanged: # commit the queue change(s) transaction.commit() finally: mutex.release() security.declarePrivate('_get_ProcessingMax') def _getProcessingMax(self,request): # Get process max timeout, checking for exact queue key, keyclass, and default, in that order. key = request['key'] key_class = key.split('_')[0] windows = self.secProcessWindows return windows.get(key, windows.get(key_class, self.secDefaultProcessWindow)) security.declarePrivate('_email_cleanup_message') def _email_cleanup_message(self,request,timeout): # email techsupport to notify that request has been forcibly been removed from the queue mailhost = self.MailHost portal = self.portal_url.getPortalObject() mfrom = portal.getProperty('email_from_address') mto = portal.getProperty('techsupport_email_address') or mfrom if mto: subject = "Request '%s' was forcibly removed from processing queue." % request['key'] messageText = "The request did not successfully complete within the required time window of %s. The request may have crashed, may be in an infinite loop, or may just be taking a really long time.\n\nThe server and pid referenced in the request need to be checked to see if the process is still active. If still active, the process may need to be killed. The QueueTool will no longer consider this an active request.\n\nThe complete request was:\n\n%s\n\n" % (timeout,str(request)) self._mailhost_send(messageText, mto, mfrom, subject) self.removeRequest(request['key']) self.processingRequests._p_changed = 1 security.declarePrivate('_mailhost_send') def _mailhost_send(self, messageText, mto, mfrom, subject): """attempt to send mail: log if fail""" portal = self.portal_url.getPortalObject() try: mailhost.send(messageText, mto, mfrom, subject) except: portal.plone_log("Can't send email:\n\nSubject:%s\n\n%s" % (subject,messageText)) security.declarePublic('ping') def ping(self): """Test method for ClockServer. Logs when called.""" self.plone_log("... QueueTool.ping() has been called ...") security.declarePublic('clockTick') def clockTick(self): """ Called on a clock event every N seconds. If work is available, a child zopectl process is launched. """ # clockTick() acquires mutex lock and is responsible for commiting the transaction. mutex.acquire() portal = self.portal_url.getPortalObject() try: if self.gotRequest(): dictRequest = self.getRequest() if dictRequest: transaction.commit() # make list change visible to child instance - other way was race condition self.launchRequest(dictRequest) portal.plone_log("... QueueTool.clockTick() has spawned a child process for key '%s' ..." % dictRequest['key']) finally: mutex.release() # removed "expired" request in processing self.cleanupProcessing() security.declarePublic('checkDepends') def checkDepends(self, request): """Check the request dependecies, in declared order. Return first unmet one. On success of all (or no declared dependencies) return 'None'""" for dep in request.get('depends',[]): try: checker = getattr(self,"%s_dep" % dep[1]) if checker(request,dep): return dep except AttributeError: self.portal_url.getPortalObject().plone_log('Skipping badly declared dependency: %s' % (dep,)) pass return None security.declarePrivate('_email_depend_fail') def _email_depend_fail(self,request,depend): # email techsupport to notify that request has been forcibly been removed from the queue mailhost = self.MailHost portal = self.portal_url.getPortalObject() mfrom = portal.getProperty('email_from_address') mto = portal.getProperty('techsupport_email_address') or mfrom if mto: subject = "Request '%s' was removed from pending queue. (dependency)" % request['key'] messageText = "The request declared a dependency that was not available at the time of processing. The QueueTool will no longer consider this an active request.\n\nThe failed dependency was:\n\n%s\n\nThe complete request was:\n\n%s\n\n" % (str(depend),str(request)) self._mailhost_send(messageText, mto, mfrom, subject) self.removeRequest(request['key'], self.pendingRequests) self.pendingRequests._p_changed = 1 security.declarePrivate('launchRequest') def launchRequest(self, dictRequest): """ Launch the input request. """ # caller must acquire mutex lock. caller is responsible for commiting the transaction. portal = self.portal_url.getPortalObject() if dictRequest is not None: key = dictRequest['key'] or None iListIndex = key is not None and self.find(key, self.processingRequests) if iListIndex is not None: # request is in the processing queue if 'timeRequestProcessed' not in dictRequest: # request has not already been started dictRequest['timeRequestProcessed'] = datetime.now() # dictRequest['ip'] = getipaddr() dictRequest['hostName'] = getfqdn() # dictRequest['pid'] = -1 key = dictRequest['key'] serverURL = dictRequest.get('serverURL',self.REQUEST['SERVER_URL']) callbackrequesthandler = dictRequest['requestHandler'] cmd = os.path.join(INSTANCE_HOME, 'bin', 'zopectl') callback = os.path.join(INSTANCE_HOME, 'Products', callbackrequesthandler) #pid = os.spawnl(os.P_NOWAIT, cmd, 'zopectl', 'run', callback, key) # call newer, better python API which allows for the stdout and stderr to be captured import subprocess f1 = open('/tmp/%s.stdout' % key, 'w') f2 = open('/tmp/%s.stderr' % key, 'w') pid = subprocess.Popen([cmd, "run", callback, key, serverURL], stdout=f1, stderr=f2).pid # child will call start() and stop() dictRequest['pid'] = pid self.processingRequests._p_changed = 1 security.declareProtected(ManagePermission, 'manage_tick') def manage_tick(self): """ZMI-useful variant of 'clockTick'. Goes back to overview page of tool. From anywhere else, use 'clockTick' instead. """ self.clockTick() self.REQUEST.RESPONSE.redirect('manage_overview') return security.declareProtected(ManagePermission, 'manage_cut_in_line') def manage_cut_in_line(self, key=""): """Move request to the head of the pendingRequests queue. """ # manage_cut_in_line() acquires mutex lock and is responsible for commiting the transaction. mutex.acquire() try: portal = self.portal_url.getPortalObject() iListIndex = self.find(key, self.pendingRequests) if iListIndex is not None and iListIndex > 0: dictRequest = self.pendingRequests[iListIndex] or None if dictRequest is not None: del self.pendingRequests[iListIndex] self.pendingRequests.insert(0, dictRequest) transaction.commit() portal.plone_log("... QueueTool.manage_cut_in_line: '%s' is cutting in line ..." % key) self.REQUEST.RESPONSE.redirect('manage_overview') finally: mutex.release() return security.declareProtected(ManagePermission, 'manage_remove') def manage_remove(self, key="", queue="pending"): """Remove request from a queue list """ # manage_cut_in_line() acquires mutex lock and is responsible for commiting the transaction. mutex.acquire() if queue == "pending": fromList = self.pendingRequests elif queue == "processing": fromList = self.processingRequests else: return try: self.removeRequest(key,fromList) portal = self.portal_url.getPortalObject() portal.plone_log("... QueueTool.manage_remove: '%s' has been removed ..." % key) self.REQUEST.RESPONSE.redirect('manage_overview') finally: mutex.release() return security.declareProtected(ManagePermission, 'getProcessingRequests') def getProcessingRequests(self): """Get process requests.""" return self.processingRequests security.declareProtected(ManagePermission, 'getPendingRequests') def getPendingRequests(self): """Get pending requests.""" return self.pendingRequests security.declarePublic('pending_count') def pending_count(self): """Return number of pending entries in the queue. Useful for monitoring.""" return len(self.getPendingRequests()) security.declarePublic('processing_count') def processing_count(self): """Return number of pending entries in the queue. Useful for monitoring.""" return len(self.getProcessingRequests())
def checkTheWorld(self): # Test constructors u = PersistentList() u0 = PersistentList(l0) u1 = PersistentList(l1) u2 = PersistentList(l2) uu = PersistentList(u) uu0 = PersistentList(u0) uu1 = PersistentList(u1) uu2 = PersistentList(u2) v = PersistentList(tuple(u)) class OtherList: def __init__(self, initlist): self.__data = initlist def __len__(self): return len(self.__data) def __getitem__(self, i): return self.__data[i] v0 = PersistentList(OtherList(u0)) vv = PersistentList("this is also a sequence") # Test __repr__ eq = self.assertEqual eq(str(u0), str(l0), "str(u0) == str(l0)") eq(repr(u1), repr(l1), "repr(u1) == repr(l1)") eq(`u2`, `l2`, "`u2` == `l2`") # Test __cmp__ and __len__ def mycmp(a, b): r = cmp(a, b) if r < 0: return -1 if r > 0: return 1 return r all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2] for a in all: for b in all: eq(mycmp(a, b), mycmp(len(a), len(b)), "mycmp(a, b) == mycmp(len(a), len(b))") # Test __getitem__ for i in range(len(u2)): eq(u2[i], i, "u2[i] == i") # Test __setitem__ uu2[0] = 0 uu2[1] = 100 try: uu2[2] = 200 except IndexError: pass else: raise TestFailed("uu2[2] shouldn't be assignable") # Test __delitem__ del uu2[1] del uu2[0] try: del uu2[0] except IndexError: pass else: raise TestFailed("uu2[0] shouldn't be deletable") # Test __getslice__ for i in range(-3, 4): eq(u2[:i], l2[:i], "u2[:i] == l2[:i]") eq(u2[i:], l2[i:], "u2[i:] == l2[i:]") for j in range(-3, 4): eq(u2[i:j], l2[i:j], "u2[i:j] == l2[i:j]") # Test __setslice__ for i in range(-3, 4): u2[:i] = l2[:i] eq(u2, l2, "u2 == l2") u2[i:] = l2[i:] eq(u2, l2, "u2 == l2") for j in range(-3, 4): u2[i:j] = l2[i:j] eq(u2, l2, "u2 == l2") uu2 = u2[:] uu2[:0] = [-2, -1] eq(uu2, [-2, -1, 0, 1], "uu2 == [-2, -1, 0, 1]") uu2[0:] = [] eq(uu2, [], "uu2 == []") # Test __contains__ for i in u2: self.failUnless(i in u2, "i in u2") for i in min(u2)-1, max(u2)+1: self.failUnless(i not in u2, "i not in u2") # Test __delslice__ uu2 = u2[:] del uu2[1:2] del uu2[0:1] eq(uu2, [], "uu2 == []") uu2 = u2[:] del uu2[1:] del uu2[:1] eq(uu2, [], "uu2 == []") # Test __add__, __radd__, __mul__ and __rmul__ #self.failUnless(u1 + [] == [] + u1 == u1, "u1 + [] == [] + u1 == u1") self.failUnless(u1 + [1] == u2, "u1 + [1] == u2") #self.failUnless([-1] + u1 == [-1, 0], "[-1] + u1 == [-1, 0]") self.failUnless(u2 == u2*1 == 1*u2, "u2 == u2*1 == 1*u2") self.failUnless(u2+u2 == u2*2 == 2*u2, "u2+u2 == u2*2 == 2*u2") self.failUnless(u2+u2+u2 == u2*3 == 3*u2, "u2+u2+u2 == u2*3 == 3*u2") # Test append u = u1[:] u.append(1) eq(u, u2, "u == u2") # Test insert u = u2[:] u.insert(0, -1) eq(u, [-1, 0, 1], "u == [-1, 0, 1]") # Test pop u = PersistentList([0, -1, 1]) u.pop() eq(u, [0, -1], "u == [0, -1]") u.pop(0) eq(u, [-1], "u == [-1]") # Test remove u = u2[:] u.remove(1) eq(u, u1, "u == u1") # Test count u = u2*3 eq(u.count(0), 3, "u.count(0) == 3") eq(u.count(1), 3, "u.count(1) == 3") eq(u.count(2), 0, "u.count(2) == 0") # Test index eq(u2.index(0), 0, "u2.index(0) == 0") eq(u2.index(1), 1, "u2.index(1) == 1") try: u2.index(2) except ValueError: pass else: raise TestFailed("expected ValueError") # Test reverse u = u2[:] u.reverse() eq(u, [1, 0], "u == [1, 0]") u.reverse() eq(u, u2, "u == u2") # Test sort u = PersistentList([1, 0]) u.sort() eq(u, u2, "u == u2") # Test extend u = u1[:] u.extend(u2) eq(u, u1 + u2, "u == u1 + u2")
def test_conflict_errors(self): """ Here we realize that conflict errors occur only occur when concurrent modifications on a particular container (with specific oid) occur concurrently. Updates can still be lost if a branch of the object tree is disconnected from the root while one of its leaves gets updated. Similarly, readCurrent() only protects a specific container of the object tree, which can still be disconnected from the root by a transaction, while its content is updated by another transaction. """ conn = self.conn root = conn.root root.stuff = PersistentList([9]) root.origin = PersistentList([3]) root.target = PersistentList([8]) root.dummy1 = PersistentList([9]) transaction.commit() # basic conflict on root # pool.apply(delete_container, args=(self.db, "dummy1")) root.dummy2 = 5 self.assertRaises(ConflictError, transaction.commit) # conflict !! self.assertRaises(TransactionFailedError, transaction.commit) # transaction broken transaction.abort() self.assertFalse(hasattr(root, "dummy2")) # rolled back # no conflict when a branch gets detached while leaf is updated container = root.stuff pool.apply(delete_container, args=(self.db, "stuff")) container[0] = 88 transaction.commit() self.assertFalse(hasattr(root, "stuff")) # update lost # without readCurrent() - lost update # root.origin = PersistentList([13]) value = root.origin pool.apply(transfer_list_value, args=(self.db, "origin", "target")) root.target = value transaction.commit() self.assertEqual(root.target, PersistentList([13])) # we lost [3] # with readCurrent() and container update - ReadConflictError raised! # root.origin = PersistentList([17]) transaction.commit() res = conn.readCurrent(root.target) # container object selected !! assert res is None # no return value expected value = root.target pool.apply(transfer_list_value, args=(self.db, "origin", "target")) root.othertarget = value self.assertRaises(Exception, transaction.commit) self.assertEqual(root.target, PersistentList([17])) # auto refreshing occurred self.assertFalse(hasattr(root, "othertarget")) # auto refreshing occurred self.assertRaises(Exception, transaction.commit) # but transaction still broken transaction.abort() transaction.commit() # now all is ok once again # with readCurrent() and container deletion - somehow lost update! # value = root.origin[0] res = conn.readCurrent(root.origin) # container object selected !! assert res is None # no return value expected pool.apply(delete_container, args=(self.db, "origin")) root.target[0] = value # we use a value whose origin has now been deleted in other thread transaction.commit() # here it's OK, the deleted object still remains in the DB history even if unreachable
def __init__(self, folder_dir): self._dir = folder_dir self.classifier = PBayes() self.hams = PersistentList() self.spams = PersistentList()
class Dispatcher: """ Sends commands to and receives results from a web browser. Dispatcher: _Computer Science_: A routine that controls the order in which input and output devices obtain access to the processing system. (source: http://dictionary.reference.com/search?q=dispatcher) In Selenium, the Dispatcher class takes commands to be executed and puts them in a queue for a web browser to read. When the browser reads this queue for a command to execute, it adds the previous command's result to the Dispatcher's result queue. """ def __init__(self): self.queue_timeout = QUEUE_TIMEOUT self._commands = PersistentList() self._results = PersistentList() # # Command queue methods # def clearCommands(self): del self._commands[:] return 'Commands cleared' def addCommand(self, command, REQUEST=None): """ Add a command to the commands queue """ self._commands.append(command) # super important to commit the change at this point... # In some methods (like webDriver and apiDriver), there is a later call # to '_p_jar.sync(), and that # call would 'roll-back' this action if we didn't commit here. get_transaction().commit() return 'Command added' def getCommand(self, REQUEST=None): """ Retreive a command from the commands queue """ if len(self._commands) != 0: response = self._commands.pop(0) else: # if the queue is empty wait to see if any commands are entered # until we're forced to time out the request. elapsed_time = 0 step = .5 # in seconds while elapsed_time <= self.queue_timeout: time.sleep(step) #in seconds elapsed_time += step # Syncronize with the ZODB in case of any recent updates. # If this is being run as a unit test, we won't have an # attribute named "_p_jar" that we can sync() if not getattr(self,'_p_jar', None): self._p_jar.sync() try: response = self._commands.pop(0) break except IndexError: response = 'ERROR: Command queue was empty' return response def getCommandQueueSize(self, REQUEST=None): """ Query the size of the commands queue """ size = len(self._commands) return size # # Result queue methods # def clearResults(self): del self._results[:] return 'Results cleared' def addResult(self, result, REQUEST=None): """ Add a result to the results queue """ self._results.append(result) # super important to commit the change at this point... # In some methods (like webDriver and apiDriver), there is a later call # to '_p_jar.sync(), and that # call would 'roll-back' this action if we didn't commit here. get_transaction().commit() return 'Result added' def getResult(self, REQUEST=None): """ Retrieve a result from the results queue """ if len(self._results) != 0: response = self._results.pop(0) else: # if the queue is empty wait to see if any results are entered # until we're forced to time out the request. elapsed_time = 0 step = .5 # in seconds while 1: time.sleep(step) #in seconds elapsed_time += step # Syncronize with the ZODB in case of any recent updates. # If this is being run as a unit test, we won't have an # attribute named "_p_jar" that we can sync() if not getattr(self,'_p_jar', None): self._p_jar.sync() try: response = self._results.pop(0) break except IndexError: if elapsed_time >= self.queue_timeout: response = 'ERROR: Result queue was empty' break return response def getResultQueueSize(self, REQUEST=None): """ Query the size of the results queue """ size = len(self._results) return size # # Misc. methods # def setTimeout(self,timeout): """ Set the timeout period in seconds that a browser or driver should wait before timing out""" self.queue_timeout = timeout return "Timeout set to %s seconds" % timeout def webDriver(self, REQUEST=None): """ Gets a command from the command queue. Also, adds a result to the result queue, unless the seleniumStart form paramter (seleniumStart=true) is present. Note: this method is usually called from a web browser as "http://<server>/selenium-driver/driver" """ if REQUEST: command_result = REQUEST.form.get('commandResult') selenium_start = REQUEST.form.get('seleniumStart') # If 'seleniumStart' is a parameter on the request, # it means this is the first time hitting the driver, # and therefore, there is no previous result to post to # the results queue. if selenium_start: self.clearResults() if command_result: self.addResult(command_result) return self.getCommand() else: return "ERROR: Missing an HTTP REQUEST" def apiDriver(self, command_string="", REQUEST=None): """ Adds a command to the command queue, and gets a result from the result queue. Note: this method is usually called from Python source code, not a web browser.""" if REQUEST: command_string = REQUEST.form.get('command_string') self.addCommand(command_string) # if test is complete, don't try retrieving a response from the browser. if command_string.find('testComplete') >= 0: return 'test complete' else: return self.getResult()
def __init__(self, parent): Base.__init__(self, parent) self.__pers_list = PersistentList()
def setCriteriaForType(self, bib_type, criteria): """update criteria for a bibliography type given""" self.duplicates_criteria[bib_type] = PersistentList(criteria) #FIXME may need to check if any change has been done self.criteriaUpdated = True return True
def ___test_huge_db_ghosting_system(): """ Interactive testcase, to demonstrate the behaviour of ZODB regarding memory management and ghost objects. Launch it with a "top"-like window opened next to it. MIGHT TRIGGER THIS WARNING: <...>Connection.py:550: UserWarning: The <class 'persistent.list.PersistentList'> object you're saving is large. (20001339 bytes.) Perhaps you're storing media which should be stored in blobs. Perhaps you're using a non-scalable data structure, such as a PersistentMapping or PersistentList. Perhaps you're storing data in objects that aren't persistent at all. In cases like that, the data is stored in the record of the containing persistent object. In any case, storing records this big is probably a bad idea. If you insist and want to get rid of this warning, use the large_record_size option of the ZODB.DB constructor (or the large-record-size option in a configuration file) to specify a larger size. Playing with this test shows that: - contants of persistent lists and mappings are really only loaded when accessed (eg. when lookup is done on them..) - non persistent types (list(), dict()) are badly treated, and remain in memory even when committed to file """ use_non_persistent_types = False PersistentList = globals()["PersistentList"] PersistentMapping = globals()["PersistentMapping"] if use_non_persistent_types: PersistentList = list PersistentMapping = dict db, connection, root = _get_huge_db_root() root["data"] = PersistentList(PersistentMapping({"toto": "tata"*random.randint(5000, 6000)}) for i in range(20000)) root["data"][0] = PersistentMapping({"toto": "tata"*5000000}) # HUGE first element print("We have a HUGE database filled with transient data now!") time.sleep(5) transaction.commit() print("We have committed the transaction!") time.sleep(5) connection.close() db.close() # --------- db, connection, root = _get_huge_db_root() print("We have reopened the huge database now!") time.sleep(5) data = root["data"] print("We have accessed data list!") time.sleep(5) var1 = data[0] print("We have accessed first element of data list!") time.sleep(5) var1 = data[0]["toto"] print("We have unghosted first element of data list!") time.sleep(5) for i in data: i # no unghosting print("We have accessed all elements of data list!") time.sleep(5) for i in data: i["toto"] # THIS removes the ghosting effect on element print("We have unghosted all elements of data list!") time.sleep(15)
def __init__(self): self._libraries = PersistentList() self._res_types = PersistentMapping() self.linkbyuid = False
class CardStack(Persistent): def __init__(self): self._v_current_card = 0 self._v_text = None self._v_cardNumber = None self.last_id = 0 self.card_list = PersistentList() self.to_be_indexed = PersistentList() self.card_store = OOBTree.OOBTree() self.hint_store = OOBTree.OOBTree() self.CreateCard(at_end=1, and_set=0) def SetHints(self, id, hints): if self.to_be_indexed[0] == id: self.hint_store[id] = hints del self.to_be_indexed[0] else: print "Not saving hints for card id", id, "because card has been deleted." def StoreText(self, id, text): self.card_store[id] = self.Compress(text) self.hint_store[id] = None if id not in self.to_be_indexed: self.to_be_indexed.append(id) get_transaction().commit() def GetText(self, id): return self.Decompress(self.card_store[id]) def DeleteCard(self): if len(self.card_list) <= 1: return id = self.card_list[self._v_current_card] del self.card_list[self._v_current_card] del self.card_store[id] del self.hint_store[id] self.SetCard(self._v_current_card) def NewCard(self): self.SaveCurrentCard() self.CreateCard() def CreateCard(self, at_end=0, text="", and_set=1): id = self.last_id + 1 self.last_id = id if at_end: self.card_list.append(id) else: self.card_list.insert(self._v_current_card + 1, id) self.StoreText(id, text) if and_set: self.SetCard(self._v_current_card + 1) def SetCard(self, card_num): if card_num >= len(self.card_list): card_num = len(self.card_list) - 1 if card_num < 0: card_num = 0 self._v_current_card = card_num id = self.card_list[card_num] card_text = self.GetText(id) self._v_text.text = card_text self._v_cardNumber.text = 'Card: ' + str(card_num + 1) print "Card", card_num + 1, "of", len(self.card_list) def SaveCurrentCard(self): text = self._v_text.text id = self.card_list[self._v_current_card] self.StoreText(id, text) def Decompress(self, text): if len(text) == 0: return text flag_index = len(text) - 1 flag = text[flag_index] if flag == chr(0): return text[0:-1] from zlib import decompress return decompress(text[0:-1]) def Compress(self, text): old_len = len(text) if old_len == 0: return text from zlib import compress start_time = time.time() compressed = compress(text, 9) elapsed_time = time.time() - start_time new_len = len(compressed) if new_len < old_len: print "compressed", old_len, "bytes by", old_len - new_len, "bytes" return compressed + chr(1) print "leaving text of len", old_len, "uncompressed." return text + chr(0) def GoNext(self, dummy=""): self.SaveCurrentCard() self.SetCard(self._v_current_card + 1) def GoPrev(self, dummy=""): self.SaveCurrentCard() self.SetCard(self._v_current_card - 1) def GoFirst(self): self.SaveCurrentCard() self.SetCard(0) def GoLast(self): self.SaveCurrentCard() self.SetCard(len(self.card_list) - 1) def ImportStack(self): """imports stacks of the format described in Managing Gigabytes Compressing and Indexing Documents and Images Second Edition, 1999 http://www.cs.mu.oz.au/mg/ source at http://www.cs.mu.oz.au/mg/mg-1.2.1.tar.gz """ wildcard = "stack files (*.stack)|*.stack|All Files (*.*)|*.*" result = dialog.openFileDialog(None, "Import which stack?", '', '', wildcard) if result.accepted: path = result.paths[0] file = open(path, "r") cards = file.read().split(str(chr(2))) for text in cards: self.CreateCard(at_end=1, text=re.sub("\r", "\n", text), and_set=0) file.close() self.SetCard(0)
def __init__(self): self.queue_timeout = QUEUE_TIMEOUT self._commands = PersistentList() self._results = PersistentList()