def __init__(self, serverUrl, debug=None, maxMemoryItems=None, maxDiskItems=None, chunkSize=200, maxHttpRequestSize=2**20, extra_post_params=None, **kwargs): """ @serverUrl: Base URL to be used to push events notifications. @maxMemoryItems: Maximum number of items to keep queued in memory. @maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all. @debug: Save the json with nice formatting. @chunkSize: maximum number of items to send in each at each HTTP POST. @maxHttpRequestSize: limits the size of encoded data for AE, the default is 1MB. """ if not serverUrl: raise config.ConfigErrors(['HttpStatusPush requires a serverUrl']) # Parameters. self.serverUrl = serverUrl self.extra_post_params = extra_post_params or {} self.debug = debug self.chunkSize = chunkSize self.lastPushWasSuccessful = True self.maxHttpRequestSize = maxHttpRequestSize if maxDiskItems != 0: # The queue directory is determined by the server url. path = ('events_' + urlparse.urlparse(self.serverUrl)[1].split(':')[0]) queue = PersistentQueue( primaryQueue=MemoryQueue(maxItems=maxMemoryItems), secondaryQueue=DiskQueue(path, maxItems=maxDiskItems)) else: path = None queue = MemoryQueue(maxItems=maxMemoryItems) # Use the unbounded method. StatusPush.__init__(self, serverPushCb=HttpStatusPush.pushHttp, queue=queue, path=path, **kwargs)
def testQueued(self): # Verify behavior when starting up with queued items on disk. os.mkdir('fake_dir') WriteFile(os.path.join('fake_dir', '3'), 'foo3') WriteFile(os.path.join('fake_dir', '5'), 'foo5') WriteFile(os.path.join('fake_dir', '8'), 'foo8') queue = PersistentQueue(MemoryQueue(3), DiskQueue('fake_dir', 5, pickleFn=str, unpickleFn=str)) self.assertEqual(['foo3', 'foo5', 'foo8'], queue.items()) self.assertEqual(3, queue.nbItems()) self.assertEqual(['foo3', 'foo5', 'foo8'], queue.popChunk())
def __init__(self, debug=None, maxMemoryItems=None, maxDiskItems=None, chunkSize=200, maxPushSize=2**20, **kwargs): """ @serverUrl: The Nats server to be used to push events notifications to. @subject: The subject to use when publishing data @maxMemoryItems: Maximum number of items to keep queued in memory. @maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all. @debug: Save the json with nice formatting. @chunkSize: maximum number of items to send in each at each PUSH. @maxPushSize: limits the size of encoded data for AE, the default is 1MB. """ # Parameters. self.debug = debug self.chunkSize = chunkSize self.lastPushWasSuccessful = True self.maxPushSize = maxPushSize if maxDiskItems != 0: # The queue directory is determined by the server url. path = 'queue_%s' % (self.eventName()) queue = PersistentQueue( primaryQueue=MemoryQueue(maxItems=maxMemoryItems), secondaryQueue=DiskQueue(path, maxItems=maxDiskItems)) else: path = None queue = MemoryQueue(maxItems=maxMemoryItems) # Use the unbounded method. StatusPush.__init__(self, serverPushCb=QueuedStatusPush._pushData, queue=queue, path=path, **kwargs)
def __init__(self, serverPushCb, queue=None, path=None, filter=True, bufferDelay=1, retryDelay=5, blackList=None): """ @serverPushCb: callback to be used. It receives 'self' as parameter. It should call self.queueNextServerPush() when it's done to queue the next push. It is guaranteed that the queue is not empty when this function is called. @queue: a item queue that implements IQueue. @path: path to save config. @filter: when True (default), removes all "", None, False, [] or {} entries. @bufferDelay: amount of time events are queued before sending, to reduce the number of push requests rate. This is the delay between the end of a request to initializing a new one. @retryDelay: amount of time between retries when no items were pushed on last serverPushCb call. @blackList: events that shouldn't be sent. """ StatusReceiverMultiService.__init__(self) # Parameters. self.queue = queue if self.queue is None: self.queue = MemoryQueue() self.queue = IndexedQueue(self.queue) self.path = path self.filter = filter self.bufferDelay = bufferDelay self.retryDelay = retryDelay if not callable(serverPushCb): raise NotImplementedError('Please pass serverPushCb parameter.') def hookPushCb(): # Update the index so we know if the next push succeed or not, don't # update the value when the queue is empty. if not self.queue.nbItems(): return self.lastIndex = self.queue.getIndex() return serverPushCb(self) self.serverPushCb = hookPushCb self.blackList = blackList # Other defaults. # IDelayedCall object that represents the next queued push. self.task = None self.stopped = False self.lastIndex = -1 self.state = {} self.state['started'] = str(datetime.datetime.utcnow()) self.state['next_id'] = 1 self.state['last_id_pushed'] = 0 # Try to load back the state. if self.path and os.path.isdir(self.path): state_path = os.path.join(self.path, 'state') if os.path.isfile(state_path): with open(state_path, 'r') as f: self.state.update(json.load(f)) if self.queue.nbItems(): # Last shutdown was not clean, don't wait to send events. self.queueNextServerPush()
def testPersistentQueue(self): self._test_helper(PersistentQueue(MemoryQueue(3), DiskQueue('fake_dir', 5)))
def testMemoryQueue(self): self._test_helper(MemoryQueue(maxItems=8))