def __init__(self, serverUrl, debug=None, maxMemoryItems=None, maxDiskItems=None, chunkSize=200, maxHttpRequestSize=2**20, **kwargs): """ @serverUrl: Base URL to be used to push events notifications. @maxMemoryItems: Maximum number of items to keep queued in memory. @maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all. @debug: Save the json with nice formatting. @chunkSize: maximum number of items to send in each at each HTTP POST. @maxHttpRequestSize: limits the size of encoded data for AE, the default is 1MB. """ # Parameters. self.serverUrl = serverUrl self.debug = debug self.chunkSize = chunkSize self.lastPushWasSuccessful = True self.maxHttpRequestSize = maxHttpRequestSize if maxDiskItems != 0: # The queue directory is determined by the server url. path = ('events_' + urlparse.urlparse(self.serverUrl)[1].split(':')[0]) queue = PersistentQueue( primaryQueue=MemoryQueue(maxItems=maxMemoryItems), secondaryQueue=DiskQueue(path, maxItems=maxDiskItems)) else: path = None queue = MemoryQueue(maxItems=maxMemoryItems) # Use the unbounded method. StatusPush.__init__(self, serverPushCb=HttpStatusPush.pushHttp, queue=queue, path=path, **kwargs)
def testQueued(self): # Verify behavior when starting up with queued items on disk. os.mkdir('fake_dir') WriteFile(os.path.join('fake_dir', '3'), 'foo3') WriteFile(os.path.join('fake_dir', '5'), 'foo5') WriteFile(os.path.join('fake_dir', '8'), 'foo8') queue = PersistentQueue(MemoryQueue(3), DiskQueue('fake_dir', 5, pickleFn=str, unpickleFn=str)) self.assertEqual(['foo3', 'foo5', 'foo8'], queue.items()) self.assertEqual(3, queue.nbItems()) self.assertEqual(['foo3', 'foo5', 'foo8'], queue.popChunk())
def __init__(self, serverIP, serverPort, debug=None, maxMemoryItems=None, maxDiskItems=None, chunkSize=200, **kwargs): """ @serverIP: IP of the autobahn server @serverPort: Port of the autobahn server @debug: Save the json with nice formatting. @maxMemoryItems: Maximum number of items to keep queued in memory. @maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all. @chunkSize: maximum number of items to send in each at each PUSH. """ if not serverIP and not serverPort: raise config.ConfigErrors( ['AutobahnStatusPush requires a serverIP and serverPort']) # Parameters. self.serverIP = serverIP self.serverPort = serverPort self.serverUrl = "ws://{0}:{1}/ws".format(serverIP, serverPort) self.debug = debug self.chunkSize = chunkSize self.lastPushWasSuccessful = True self.protocol = None self.factory = None if maxDiskItems != 0: # The queue directory is determined by the server url. path = ('events_' + urlparse.urlparse(self.serverUrl)[1].split(':')[0]) queue = PersistentQueue( primaryQueue=MemoryQueue(maxItems=maxMemoryItems), secondaryQueue=DiskQueue(path, maxItems=maxDiskItems)) else: path = None queue = MemoryQueue(maxItems=maxMemoryItems) self.connectToAutobahn() # Use the unbounded method. StatusPush.__init__(self, serverPushCb=AutobahnStatusPush.pushHttp, queue=queue, path=path, **kwargs)
def __init__(self, debug=None, maxMemoryItems=None, maxDiskItems=None, chunkSize=200, maxPushSize=2**20, **kwargs): """ @serverUrl: The Nats server to be used to push events notifications to. @subject: The subject to use when publishing data @maxMemoryItems: Maximum number of items to keep queued in memory. @maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all. @debug: Save the json with nice formatting. @chunkSize: maximum number of items to send in each at each PUSH. @maxPushSize: limits the size of encoded data for AE, the default is 1MB. """ # Parameters. self.debug = debug self.chunkSize = chunkSize self.lastPushWasSuccessful = True self.maxPushSize = maxPushSize if maxDiskItems != 0: # The queue directory is determined by the server url. path = 'queue_%s' % (self.eventName()) queue = PersistentQueue( primaryQueue=MemoryQueue(maxItems=maxMemoryItems), secondaryQueue=DiskQueue(path, maxItems=maxDiskItems)) else: path = None queue = MemoryQueue(maxItems=maxMemoryItems) # Use the unbounded method. StatusPush.__init__(self, serverPushCb=QueuedStatusPush._pushData, queue=queue, path=path, **kwargs)
def testPersistentQueue(self): self._test_helper(PersistentQueue(MemoryQueue(3), DiskQueue('fake_dir', 5)))
def testDiskQueue(self): self._test_helper(DiskQueue('fake_dir', maxItems=8))