def start(self, force=False, inprocess=False): if inprocess: server = MessageServer(self._address, self.storeLocal, self._pidFile, self.echo) for address in self.forwardAddresses: client = MessageServerClient(address) server.forwardClients.add(client) server.start() return if self.isRunning and not force: printInDebugMode('Cannot start message server, already running') return command = self._createStartCommand() printInDebugMode('Starting message server with command: %s' % command) j.system.process.runDaemon(command) remainingSeconds = self._START_TIMEOUT time.sleep(5) while not self.isRunning: printInDebugMode('Waiting %d more seconds for message server to be' ' started' % remainingSeconds) time.sleep(1) remainingSeconds -= 1 if not remainingSeconds: raise RuntimeError('Failed to start message server') printInDebugMode('Started message server')
def stop(self, force=False): if not self.isRunning and not force: printInDebugMode('Cannot stop message server, not running') return pid = self._getPidFromPidFile() if force: stopSignal = signal.SIGKILL else: stopSignal = signal.SIGTERM printInDebugMode('Stopping message server') j.system.process.kill(pid, sig=stopSignal) remainingSeconds = self._STOP_TIMEOUT while self.isRunning: printInDebugMode('Waiting %d more seconds for message server to be' ' stopped' % remainingSeconds) time.sleep(1) remainingSeconds -= 1 if not remainingSeconds: raise RuntimeError('Failed to stop message server') j.system.fs.remove(self._pidFile) printInDebugMode('Stopped message server')
def _send(self, message, receiveTimeout=_RECEIVE_TIMEOUT): ''' Tries to send a message to the server and waits a fixed amount of seconds for it to answer. @param message: message to send to the server @type message: str @param receiveTimeout: max time in seconds in which the server should respond @type receiveTimeout: int ''' if not self._isConnected: self._connect() try: self._socket.send(message) except Exception, e: self._disconnect() printInDebugMode('Couldn\'t send message, unexpected exception' ' while sending to server: %s' % e)
def _disconnect(self): '''Disconnects the client if it isn't already''' if not self._isConnected: printInDebugMode( 'Can\'t disconnect from %s, already disconnected' % self._address) return printInDebugMode('Disconnecting from %s' % self._address) self._socket.close() self._context.term() printInDebugMode('Disconnected from %s' % self._address)
def _processQueue(self): ''' Tries to process as many as possible queued messages. A message buffer is updated on every attempt. This buffer is then flattened to a single message. This reduces the amount of roundtrips with the server. When a message is not send succesfully a series of retry attempts will be completed before aborting. After every failed attempt the client backs off for a fixed amound of seconds. @return: True when a part of the queue is succesfully processes, False otherwise @rtype: bool ''' maxAttempts = len(self._BACKOFF_INTERVALS) + 1 isSucces = False buffer = list() for attempt in range(1, maxAttempts): buffer = self._updateBuffer(buffer) message = ''.join( buffer) # The message is actually a batch of messages. isSucces = self._send(message) if isSucces: break else: interval = self._BACKOFF_INTERVALS[attempt - 1] printInDebugMode( '''\ Failed to send message to %(address)s in %(attempts)d attempt(s)' Retrying to send message to %(address)s in %(interval)d seconds''' % { 'address': self._address, 'attempts': attempt, 'interval': interval }) gevent.sleep(interval) if isSucces: printInDebugMode( 'Successfully sent message to %s in %d attempt(s)' % (self._address, attempt)) else: printInDebugMode('Failed to send message to %s in %d attempt(s)' % (self._address, attempt)) return isSucces
def _connect(self): '''Connects the client if it isn't already''' if self._isConnected: printInDebugMode('Can\'t connect to %s, already connected' % self._address) return printInDebugMode('Connecting to %s' % self._address) self._context = zmq.Context(2) self._socket = self._context.socket(zmq.REQ) self._socket.setsockopt(zmq.LINGER, 0) self._socket.connect(self._address) printInDebugMode('Connected to %s' % self._address)
class MessageServerClient(object): _BACKOFF_INTERVALS = (1, 1, 1, 5, 10, 30, 60) _MAX_BUFFER_SIZE = 1000 _RECEIVE_TIMEOUT = 2 def __init__(self, address): ''' MessageServerClient constructor. @param address: address to send the messages to @type address: str ''' self._address = 'tcp://%s' % address self._socket = None self._context = None self._queue = Queue() self._processQueueInLoop() @property def address(self): return self._address def ping(self): ''' Sends a ping message to the server and checks if it answers. @return: True when the server answered, False otherwise @rtype: bool ''' return self._send('ping', receiveTimeout=1) def send(self, message): ''' Puts a message in the queue and immediately after that yields. This will give the worker which is running in a Greenlet the time to process the queue. @param message: message to send to the server @type message: str ''' self._queue.put(message) gevent.sleep() @property def _isConnected(self): ''' Returns a flag that states if the client is connected or not. @return: flag that states if the client is connected or not @rtype: bool ''' if self._context: return not self._context.closed else: return False def _connect(self): '''Connects the client if it isn't already''' if self._isConnected: printInDebugMode('Can\'t connect to %s, already connected' % self._address) return printInDebugMode('Connecting to %s' % self._address) self._context = zmq.Context(2) self._socket = self._context.socket(zmq.REQ) self._socket.setsockopt(zmq.LINGER, 0) self._socket.connect(self._address) printInDebugMode('Connected to %s' % self._address) def _disconnect(self): '''Disconnects the client if it isn't already''' if not self._isConnected: printInDebugMode( 'Can\'t disconnect from %s, already disconnected' % self._address) return printInDebugMode('Disconnecting from %s' % self._address) self._socket.close() self._context.term() printInDebugMode('Disconnected from %s' % self._address) def _processQueue(self): ''' Tries to process as many as possible queued messages. A message buffer is updated on every attempt. This buffer is then flattened to a single message. This reduces the amount of roundtrips with the server. When a message is not send succesfully a series of retry attempts will be completed before aborting. After every failed attempt the client backs off for a fixed amound of seconds. @return: True when a part of the queue is succesfully processes, False otherwise @rtype: bool ''' maxAttempts = len(self._BACKOFF_INTERVALS) + 1 isSucces = False buffer = list() for attempt in range(1, maxAttempts): buffer = self._updateBuffer(buffer) message = ''.join( buffer) # The message is actually a batch of messages. isSucces = self._send(message) if isSucces: break else: interval = self._BACKOFF_INTERVALS[attempt - 1] printInDebugMode( '''\ Failed to send message to %(address)s in %(attempts)d attempt(s)' Retrying to send message to %(address)s in %(interval)d seconds''' % { 'address': self._address, 'attempts': attempt, 'interval': interval }) gevent.sleep(interval) if isSucces: printInDebugMode( 'Successfully sent message to %s in %d attempt(s)' % (self._address, attempt)) else: printInDebugMode('Failed to send message to %s in %d attempt(s)' % (self._address, attempt)) return isSucces def _processQueueInLoop(self): ''' Spawns a worker in a Greenlet that processes the queue when possible. ''' def worker(): while True: self._processQueue() gevent.spawn(worker) def _updateBuffer(self, buffer): ''' Updates a buffer by moving as much messages as possible from the queue to the buffer. This method will block if the queue is empty. Once a message is put in the queue the buffer will be updated. @param buffer: buffer to update @type buffer: set() @return: updated buffer @rtype: set() ''' while len(buffer) < self._MAX_BUFFER_SIZE: # The get call on the queue will block until a message is put in the # queue. message = self._queue.get() buffer.append(message) if self._queue.empty(): break return buffer def _send(self, message, receiveTimeout=_RECEIVE_TIMEOUT): ''' Tries to send a message to the server and waits a fixed amount of seconds for it to answer. @param message: message to send to the server @type message: str @param receiveTimeout: max time in seconds in which the server should respond @type receiveTimeout: int ''' if not self._isConnected: self._connect() try: self._socket.send(message) except Exception, e: self._disconnect() printInDebugMode('Couldn\'t send message, unexpected exception' ' while sending to server: %s' % e) result = None timeout = Timeout(receiveTimeout) timeout.start() try: result = self._socket.recv() except Timeout, timeoutException: if timeoutException == timeout: printInDebugMode( 'Couldn\'t send message, server response timed' ' out after %d seconds' % receiveTimeout) self._disconnect()
result = None timeout = Timeout(receiveTimeout) timeout.start() try: result = self._socket.recv() except Timeout, timeoutException: if timeoutException == timeout: printInDebugMode( 'Couldn\'t send message, server response timed' ' out after %d seconds' % receiveTimeout) self._disconnect() except Exception, e: printInDebugMode('Couldn\'t send message, unexpected exception' ' while receiving response from server: %s' % e) self._disconnect() finally: timeout.cancel() return result == '1' def __del__(self): ''' Makes sure that the client is disconnected properly before being deleted. ''' if self._isConnected: self._disconnect()