class PubsubHandler:
def __init__(self,
uri,
exchangeName=None,
exchangeType='direct',
routingKey=None,
otherKeys=[],
applicationId=None,
verbose=False,
subscriberName=None,
recyclebinName=None,
redeliveredLimit=3):
self.__engine = Engine(
**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': exchangeType,
'routingKey': routingKey,
'otherKeys': otherKeys,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
self.__listener = None
if subscriberName is not None and type(subscriberName) is str:
self.__subscriberName = subscriberName
self.__executor.assertQueue(self.__subscriberName)
else:
self.__subscriberName = None
if recyclebinName is not None and type(recyclebinName) is str:
self.__recyclebinName = recyclebinName
self.__executor.assertQueue(self.__recyclebinName)
else:
self.__recyclebinName = None
if redeliveredLimit is not None and type(redeliveredLimit) is int:
self.__redeliveredLimit = 0 if (
redeliveredLimit < 0) else redeliveredLimit
else:
self.__recyclebinName = None
def publish(self, data, opts={}, routingKey=None):
opts = {} if opts is None else opts
if 'requestId' not in opts or opts['requestId'] is None:
opts['requestId'] = Util.getUUID()
properties = {'headers': opts}
override = {}
if routingKey is not None and type(routingKey) is str:
override['routingKey'] = routingKey
self.__engine.produce(message=data,
properties=properties,
override=override)
def subscribe(self, pubsubListener):
self.__listener = pubsubListener if (
self.__listener is None) else self.__listener
if self.__listener is None:
raise ParameterError('Subscriber callback should not be None')
elif self.__listener != pubsubListener:
raise ParameterError(
'PubsubHandler only supports single Subscriber callback')
def opflowListener(channel, method, properties, body, replyToName):
headers = properties.headers
requestId = Util.getRequestId(headers)
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Request[%s] - subscribe() receive a message' %
requestId)
try:
pubsubListener(body, headers)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
'Request[%s] - subscribe() has processed successfully'
% requestId)
except Exception as exception:
redeliveredCount = 0
if 'redeliveredCount' in headers and type(
headers['redeliveredCount']) is int:
redeliveredCount = headers['redeliveredCount']
redeliveredCount += 1
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
'Request[%s] - redeliveredCount: %s/%s' %
(requestId, redeliveredCount, self.__redeliveredLimit))
headers['redeliveredCount'] = redeliveredCount
props = properties
if redeliveredCount <= self.__redeliveredLimit:
self.__sendToQueue(content=body,
properties=props,
queueName=self.__subscriberName,
channel=channel)
else:
if self.__recyclebinName is not None:
self.__sendToQueue(content=body,
properties=props,
queueName=self.__recyclebinName,
channel=channel)
return True
return self.__engine.consume(opflowListener, {
'noAck': True,
'queueName': self.__subscriberName
})
def close(self):
if self.__engine is not None:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Engine is closing')
self.__engine.close()
def retain(self):
_CONSUMING_LOOP_INTERVAL = 1
if self.__engine is not None:
while self.__engine.consumingLoop is not None and self.__engine.consumingLoop.is_alive(
):
if self.__engine.verbose and logger.isEnabledFor(
logging.DEBUG):
logger.debug('consumingLoop interval: %s second(s)' %
_CONSUMING_LOOP_INTERVAL)
self.__engine.consumingLoop.join(_CONSUMING_LOOP_INTERVAL)
def __sendToQueue(self, content, properties, queueName, channel):
try:
channel.basic_publish(exchange='',
routing_key=queueName,
body=content,
properties=properties)
except:
raise OperationError('sendToQueue(%s) has been failed' % queueName)
def __copyProperties(self, properties):
props = pika.spec.BasicProperties(**dict(properties, headers=headers))
return props
@property
def executor(self):
return self.__executor
class RpcWorker:
def __init__(self,
uri,
exchangeName=None,
routingKey=None,
applicationId=None,
operatorName=None,
responseName=None,
verbose=False):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Constructor begin ...')
self.__engine = Engine(
**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': 'direct',
'routingKey': routingKey,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
if operatorName is not None and type(operatorName) is str:
self.__operatorName = operatorName
self.__executor.assertQueue(self.__operatorName)
else:
raise ConstructorError('operatorName should not be empty')
if responseName is not None and type(responseName) is str:
self.__responseName = responseName
self.__executor.assertQueue(self.__responseName)
else:
self.__responseName = None
self.__middlewares = []
self.__consumerInfo = None
if logger.isEnabledFor(logging.DEBUG): logger.debug('Constructor end!')
def process(self, routineId, callback):
routineIdChecker = None
if (routineId is None):
routineIdChecker = lambda (varId): True
elif (type(routineId) is str):
routineIdChecker = lambda (varId): (varId == routineId)
elif (type(routineId) is list):
routineIdChecker = lambda (varId): (varId in routineId)
if (callable(routineIdChecker)):
self.__middlewares.append({
'checker': routineIdChecker,
'listener': callback
})
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Middlewares length: %s' %
(len(self.__middlewares)))
else:
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
'Invalid checker (routineIdChecker is not callable), skipped'
)
if (self.__consumerInfo is not None):
return self.__consumerInfo
def workerCallback(channel, method, properties, body, replyToName):
response = RpcResponse(channel, properties, method.consumer_tag,
body, replyToName)
routineId = Util.getRoutineId(properties.headers)
count = 0
for middleware in self.__middlewares:
if (middleware['checker'](routineId)):
count += 1
middleware['listener'](body, properties.headers, response)
if logger.isEnabledFor(logging.DEBUG):
logger.debug('middleware matched, invoke callback')
else:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('middleware not matched, skipped')
return (count > 0)
self.__consumerInfo = self.__engine.consume(
workerCallback, {
'binding': True,
'queueName': self.__operatorName,
'replyTo': self.__responseName
})
return self.__consumerInfo
def close(self):
if self.__consumerInfo is not None:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Consumer is cancelling')
self.__engine.cancelConsumer(self.__consumerInfo)
if self.__engine is not None:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Engine is closing')
self.__engine.close()
def retain(self):
_CONSUMING_LOOP_INTERVAL = 1
if self.__engine is not None:
while self.__engine.consumingLoop is not None and self.__engine.consumingLoop.is_alive(
):
if self.__engine.verbose and logger.isEnabledFor(
logging.DEBUG):
logger.debug('consumingLoop interval: %s second(s)' %
_CONSUMING_LOOP_INTERVAL)
self.__engine.consumingLoop.join(_CONSUMING_LOOP_INTERVAL)
@property
def executor(self):
if self.__executor is None:
self.__executor = Executor(engine=self.__engine)
return self.__executor
class RpcMaster:
def __init__(self, uri, exchangeName=None, routingKey=None, applicationId=None,
responseName=None, verbose=False,
monitorEnabled=True, monitorId=None, monitorInterval=None, monitorTimeout=None):
if logger.isEnabledFor(logging.DEBUG): logger.debug('Constructor begin ...')
self.__lock = threading.RLock()
self.__idle = threading.Condition(self.__lock)
self.__engine = Engine(**{
'uri': uri,
'exchangeName': exchangeName,
'exchangeType': 'direct',
'routingKey': routingKey,
'applicationId': applicationId,
'verbose': verbose
})
self.__executor = Executor(engine=self.__engine)
self.__tasks = {}
self.__timeoutHandler = None
self.__responseConsumer = None
if responseName is not None and type(responseName) is str:
self.__responseName = responseName
self.__executor.assertQueue(self.__responseName)
else:
self.__responseName = None
if monitorEnabled is not None and type(monitorEnabled) is bool:
self.__monitorEnabled = monitorEnabled
else:
self.__monitorEnabled = True
if monitorId is not None and type(monitorId) is str:
self.__monitorId = monitorId
else:
self.__monitorId = Util.getUUID()
if monitorInterval is not None and type(monitorInterval) is int:
self.__monitorInterval = monitorInterval
else:
self.__monitorInterval = 1
if monitorTimeout is not None and type(monitorTimeout) is int:
self.__monitorTimeout = monitorTimeout
else:
self.__monitorTimeout = 0
def request(self, routineId, content, options=None):
if logger.isEnabledFor(logging.DEBUG): logger.debug('request() is invoked')
if (options is None): options = {}
forked = ('mode' in options and options['mode'] == 'forked')
if self.__monitorEnabled and self.__timeoutHandler is None:
self.__timeoutHandler = TimeoutHandler(tasks=self.__tasks,
monitorId=self.__monitorId,
interval=self.__monitorInterval,
timeout=self.__monitorTimeout)
self.__timeoutHandler.start()
if logger.isEnabledFor(logging.DEBUG):
logger.debug('timeoutHandler[%s] has been started' % (self.__monitorId))
consumerInfo = None
if forked:
consumerInfo = self.__initResponseConsumer(True)
else:
if (self.__responseConsumer is None):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('request() create new ResponseConsumer')
self.__responseConsumer = self.__initResponseConsumer(False)
consumerInfo = self.__responseConsumer
taskId = Util.getUUID()
if logger.isEnabledFor(logging.DEBUG):
logger.debug('request() - new taskId: %s' % (taskId))
def completeListener():
self.__lock.acquire()
try:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('completeListener will be invoked')
del self.__tasks[taskId]
if len(self.__tasks) == 0:
if forked:
self.__engine.cancelConsumer(consumerInfo)
self.__idle.notify()
if logger.isEnabledFor(logging.DEBUG): logger.debug('tasks is empty')
finally:
self.__lock.release()
if (routineId is not None):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('request() - routineId: %s' % (routineId))
options['routineId'] = routineId
task = RpcRequest(options, completeListener)
self.__tasks[taskId] = task
headers = { 'routineId': task.routineId, 'requestId': task.requestId }
properties = { 'headers': headers, 'correlation_id': taskId }
if not consumerInfo['fixedQueue']:
properties['reply_to'] = consumerInfo['queueName']
self.__engine.produce(message=content, properties=properties)
return task
def __initResponseConsumer(self, forked=False):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('__initResponseConsumer() - invoked with forked: %s' % forked)
def callback(channel, method, properties, body, replyToName):
taskId = properties.correlation_id
if logger.isEnabledFor(logging.DEBUG):
logger.debug('responseConsumer - task[%s] received data' % (taskId))
if (taskId not in self.__tasks):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('responseConsumer - task[%s] not found, skipped' % (taskId))
return
task = self.__tasks[taskId]
task.push({ 'content': body, 'headers': properties.headers })
if logger.isEnabledFor(logging.DEBUG):
logger.debug('responseConsumer - task[%s] message enqueued' % (taskId))
return True
options = { 'binding': False, 'prefetch': 1 }
if (not forked):
options['queueName'] = self.__responseName
options['consumerLimit'] = 1
options['forceNewChannel'] = False
if logger.isEnabledFor(logging.DEBUG):
logger.debug('__initResponseConsumer() - options: %s' % options)
return self.__engine.consume(callback, options)
def close(self):
self.__lock.acquire()
try:
while len(self.__tasks) > 0: self.__idle.wait()
if self.__responseConsumer is not None:
self.__engine.cancelConsumer(self.__responseConsumer)
if self.__timeoutHandler is not None:
self.__timeoutHandler.stop()
self.__engine.close()
finally:
self.__lock.release()
def retain(self):
if self.__timeoutHandler is not None:
while self.__timeoutHandler.timer is not None and self.__timeoutHandler.timer.is_alive():
self.__timeoutHandler.timer.join(1)
if self.__engine is not None:
while self.__engine.consumingLoop is not None and self.__engine.consumingLoop.is_alive():
self.__engine.consumingLoop.join(1)
@property
def executor(self):
return self.__executor