def __init__(self,
serverUrl,
debug=None,
maxMemoryItems=None,
maxDiskItems=None,
chunkSize=200,
maxHttpRequestSize=2**20,
extra_post_params=None,
**kwargs):
"""
@serverUrl: Base URL to be used to push events notifications.
@maxMemoryItems: Maximum number of items to keep queued in memory.
@maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't
use disk at all.
@debug: Save the json with nice formatting.
@chunkSize: maximum number of items to send in each at each HTTP POST.
@maxHttpRequestSize: limits the size of encoded data for AE, the default
is 1MB.
"""
if not serverUrl:
raise config.ConfigErrors(['HttpStatusPush requires a serverUrl'])
# Parameters.
self.serverUrl = serverUrl
self.extra_post_params = extra_post_params or {}
self.debug = debug
self.chunkSize = chunkSize
self.lastPushWasSuccessful = True
self.maxHttpRequestSize = maxHttpRequestSize
if maxDiskItems != 0:
# The queue directory is determined by the server url.
path = ('events_' +
urlparse.urlparse(self.serverUrl)[1].split(':')[0])
queue = PersistentQueue(
primaryQueue=MemoryQueue(maxItems=maxMemoryItems),
secondaryQueue=DiskQueue(path, maxItems=maxDiskItems))
else:
path = None
queue = MemoryQueue(maxItems=maxMemoryItems)
# Use the unbounded method.
StatusPush.__init__(self,
serverPushCb=HttpStatusPush.pushHttp,
queue=queue,
path=path,
**kwargs)
def testQueued(self):
# Verify behavior when starting up with queued items on disk.
os.mkdir('fake_dir')
WriteFile(os.path.join('fake_dir', '3'), 'foo3')
WriteFile(os.path.join('fake_dir', '5'), 'foo5')
WriteFile(os.path.join('fake_dir', '8'), 'foo8')
queue = PersistentQueue(MemoryQueue(3),
DiskQueue('fake_dir', 5, pickleFn=str, unpickleFn=str))
self.assertEqual(['foo3', 'foo5', 'foo8'], queue.items())
self.assertEqual(3, queue.nbItems())
self.assertEqual(['foo3', 'foo5', 'foo8'], queue.popChunk())
def __init__(self,
debug=None,
maxMemoryItems=None,
maxDiskItems=None,
chunkSize=200,
maxPushSize=2**20,
**kwargs):
"""
@serverUrl: The Nats server to be used to push events notifications to.
@subject: The subject to use when publishing data
@maxMemoryItems: Maximum number of items to keep queued in memory.
@maxDiskItems: Maximum number of items to buffer to disk, if 0, doesn't use disk at all.
@debug: Save the json with nice formatting.
@chunkSize: maximum number of items to send in each at each PUSH.
@maxPushSize: limits the size of encoded data for AE, the default is 1MB.
"""
# Parameters.
self.debug = debug
self.chunkSize = chunkSize
self.lastPushWasSuccessful = True
self.maxPushSize = maxPushSize
if maxDiskItems != 0:
# The queue directory is determined by the server url.
path = 'queue_%s' % (self.eventName())
queue = PersistentQueue(
primaryQueue=MemoryQueue(maxItems=maxMemoryItems),
secondaryQueue=DiskQueue(path, maxItems=maxDiskItems))
else:
path = None
queue = MemoryQueue(maxItems=maxMemoryItems)
# Use the unbounded method.
StatusPush.__init__(self,
serverPushCb=QueuedStatusPush._pushData,
queue=queue,
path=path,
**kwargs)
def __init__(self,
serverPushCb,
queue=None,
path=None,
filter=True,
bufferDelay=1,
retryDelay=5,
blackList=None):
"""
@serverPushCb: callback to be used. It receives 'self' as parameter. It
should call self.queueNextServerPush() when it's done to queue the next
push. It is guaranteed that the queue is not empty when this function is
called.
@queue: a item queue that implements IQueue.
@path: path to save config.
@filter: when True (default), removes all "", None, False, [] or {}
entries.
@bufferDelay: amount of time events are queued before sending, to
reduce the number of push requests rate. This is the delay between the
end of a request to initializing a new one.
@retryDelay: amount of time between retries when no items were pushed on
last serverPushCb call.
@blackList: events that shouldn't be sent.
"""
StatusReceiverMultiService.__init__(self)
# Parameters.
self.queue = queue
if self.queue is None:
self.queue = MemoryQueue()
self.queue = IndexedQueue(self.queue)
self.path = path
self.filter = filter
self.bufferDelay = bufferDelay
self.retryDelay = retryDelay
if not callable(serverPushCb):
raise NotImplementedError('Please pass serverPushCb parameter.')
def hookPushCb():
# Update the index so we know if the next push succeed or not, don't
# update the value when the queue is empty.
if not self.queue.nbItems():
return
self.lastIndex = self.queue.getIndex()
return serverPushCb(self)
self.serverPushCb = hookPushCb
self.blackList = blackList
# Other defaults.
# IDelayedCall object that represents the next queued push.
self.task = None
self.stopped = False
self.lastIndex = -1
self.state = {}
self.state['started'] = str(datetime.datetime.utcnow())
self.state['next_id'] = 1
self.state['last_id_pushed'] = 0
# Try to load back the state.
if self.path and os.path.isdir(self.path):
state_path = os.path.join(self.path, 'state')
if os.path.isfile(state_path):
with open(state_path, 'r') as f:
self.state.update(json.load(f))
if self.queue.nbItems():
# Last shutdown was not clean, don't wait to send events.
self.queueNextServerPush()
def testPersistentQueue(self):
self._test_helper(PersistentQueue(MemoryQueue(3),
DiskQueue('fake_dir', 5)))
def testMemoryQueue(self):
self._test_helper(MemoryQueue(maxItems=8))
