def close(self, force=True):
closed = False
try:
closed = self.adapater.close(force)
except Exception as e:
log.error('Exception in adapater.close {e}', e=str(e))
return closed
def listen(self):
pingProcess = multiprocessing.Process(
target=self._createZmqPingServers,
args=(self._port, ),
name="Ping Servers Process")
pingProcess.daemon = True
pingProcess.start()
for _ in range(self._num_threads):
server = ZMQServer(self._context, self._replyFunc,
self._url_worker)
server.start()
self._instances.append(server)
try:
self._device = zmq.devices.ThreadDevice(zmq.QUEUE, zmq.ROUTER,
zmq.DEALER)
self._device.context_factory = lambda: self._context
self._device.bind_in(self._url_client)
self._device.setsockopt_in(zmq.LINGER, 0)
self._device.bind_out(self._url_worker)
self._device.setsockopt_out(zmq.LINGER, 0)
self._device.start()
except Exception as e:
log.error('zmq.device failed with {e}', e=str(e))
raise
def _createZmqPingServers(self, port):
try:
pingContext = zmq.Context()
url_worker = "inproc://ping_workers"
url_client = "tcp://*:" + str(port + 1)
def createDealerRouter(url_client, url_worker, context):
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
return (clients, workers)
clients, workers = createDealerRouter(url_client=url_client,
url_worker=url_worker,
context=pingContext)
log.info('clients: {c}, workers: {w}',
c=clients.get(zmq.LAST_ENDPOINT),
w=workers.get(zmq.LAST_ENDPOINT))
log.info(
'Creating {num_threads} ZMQ Ping Servers on port {port}. pid: {pid}',
port=url_client,
num_threads=self._num_ping_threads,
pid=os.getpid())
for i in range(self._num_ping_threads):
server = ZMQPingServer(pingContext, url_worker,
'Ping-Thread-' + str(i))
server.start()
zmq.device(zmq.QUEUE, clients, workers)
except Exception as e:
log.error(e)
def request(self):
self.socket.send(self.content)
result = self.socket.poll(self.timeout)
if (result):
message = self.socket.recv_multipart()
return message
err = 'request timed out ({timeout}) to {conn}'.format(
timeout=self.timeout, conn=self.connStr)
log.error(err)
raise Exception(err)
def ping(self):
self.ping_socket.send(consts.zmq.ping)
result = self.ping_socket.poll(self.networkTimeout)
if (result):
message = self.ping_socket.recv()
if (message == consts.zmq.pong):
return True
err = 'ping timed out ({timeout}) to {conn}'.format(
timeout=self.networkTimeout, conn=self.pingConnStr)
log.error(err)
raise Exception(err)
def onMessage(self, messageFlowPattern, msg, origin, sendMessageId=None):
self.threadLocalStorage.sendMessageId = sendMessageId
self.threadLocalStorage.messageFlowPattern = messageFlowPattern
if (not self._inputListener):
log.error('no input listeners on _onMessage method')
return
for listener in self._inputListener:
try:
listener(msg, origin)
except Exception as e:
log.error("hkube_api message listener threw exception: {e}",
e=str(e))
self.threadLocalStorage.messageFlowPattern = []
def _invokeAlgorithm(self, msg, origin):
if not (self.originalAlgorithm.get('init') is None):
self.originalAlgorithm['init'](msg)
# TODO should init be called upon every message
options = {}
options.update(self.options)
options['streamInput'] = {'message': msg, 'origin': origin}
try:
result = self.originalAlgorithm['start'](options, self._hkubeApi)
if (self.options['childs']):
self._hkubeApi.sendMessage(result)
except Exception as e:
log.error('statelessWrapper error, {e}', e=str(e))
self.error = e
def sendError(self, error):
try:
log.error(error)
self._wsc.send({
'command': messages.outgoing.error,
'error': {
'code': 'Failed',
'message': self._errorMsg(error)
}
})
if (self._job.isStreaming):
self._hkubeApi.stopStreaming(False)
except Exception as e:
log.error(e)
def onMessage(self, messageFlowPattern, header, msg):
start = time.time()
decodedMsg = self._encoding.decode(header=header, value=msg)
for listener in self.messageListeners:
try:
listener(messageFlowPattern, decodedMsg,
self.messageOriginNodeName)
except Exception as e:
log.error('Error during MessageListener onMessage {e}',
e=str(e))
end = time.time()
duration = float((end - start) * 1000)
return self._encoding.encode({'duration': round(duration, 4)},
plainEncode=True)
def _checkQueueSize(self, event):
if (self._job.isStreaming):
if (self.streamingManager.messageProducer):
try:
log.info('Messages left in queue on {event}={queue}',
event=event,
queue=str(
len(self.streamingManager.messageProducer.
adapter.messageQueue.queue)))
except Exception:
log.error(
'Failed to print number of messages left in queue on {event}',
event=event)
else:
log.info('MessageProducer already None on {event}',
event=event)
def _exit(self, options):
try:
self._dataServer and self._dataServer.shutDown()
self._wsc.shutDown()
method = self._getMethod('exit')
if (method is not None):
method(options)
self._checkQueueSize(event='exit')
option = options if options is not None else dict()
code = option.get('exitCode', 0)
self._active = False
log.info('Got exit command. Exiting with code {code}', code=code)
sys.exit(code)
except Exception as e:
log.error('Got error during exit: {e}', e=e)
# pylint: disable=protected-access
os._exit(0)
def run(self):
self._socket = self._context.socket(zmq.REP)
self._socket.setsockopt(zmq.LINGER, 0)
self._socket.connect(self._workerUrl)
while self._active:
try:
events = self._socket.poll(timeout=1000)
if (events == 0):
continue
message = self._socket.recv()
if(message == consts.zmq.ping):
self._socket.send(consts.zmq.pong)
except Exception as e:
log.error('socket closed: {e}', e=str(e))
break
log.info('ZmqPingServer run loop exit')
self.close()
def loadAlgorithm(self, options):
try:
cwd = os.getcwd()
algOptions = options.algorithm
package = algOptions["path"]
entry = algOptions["entryPoint"]
entryPoint = Algorunner._getEntryPoint(entry)
__import__(package)
os.chdir('{cwd}/{package}'.format(cwd=cwd, package=package))
log.info('loading {entry}', entry=entry)
mod = importlib.import_module(
'.{entryPoint}'.format(entryPoint=entryPoint), package=package)
log.info('algorithm code loaded')
for k, v in methods.items():
methodName = k
method = v
isMandatory = method["mandatory"]
try:
self._originalAlgorithm[methodName] = getattr(
mod, methodName)
# fix start if it has only one argument
if methodName == 'start' and self._originalAlgorithm[
'start'].__code__.co_argcount == 1:
self._originalAlgorithm[
'startOrig'] = self._originalAlgorithm['start']
self._originalAlgorithm[
'start'] = lambda args, api: self._originalAlgorithm[
'startOrig'](args)
log.info('found method {methodName}',
methodName=methodName)
except Exception as e:
mandatory = "mandatory" if isMandatory else "optional"
error = 'unable to find {mandatory} method {methodName}'.format(
mandatory=mandatory, methodName=methodName)
if (isMandatory):
raise Exception(error)
log.warning(error)
self._wrapStateless()
self._tracer = Tracer(options.tracer)
except Exception as e:
self._loadAlgorithmError = self._errorMsg(e)
traceback.print_exc()
log.error(e)
def sendMessage(self, msg, flowName=None):
if (self.messageProducer is None):
raise Exception(
'Trying to send a message from a none stream pipeline or after close had been applied on algorithm'
)
if (self.messageProducer.nodeNames):
parsedFlow = None
if (flowName is None):
if hasattr(self.threadLocalStorage, 'messageFlowPattern'
) and self.threadLocalStorage.messageFlowPattern:
parsedFlow = self.threadLocalStorage.messageFlowPattern
else:
if (self.defaultFlow is None):
raise Exception("Streaming default flow is None")
flowName = self.defaultFlow
if not (parsedFlow):
parsedFlow = self.parsedFlows.get(flowName)
if (parsedFlow is None):
raise Exception("No such flow " + flowName)
self.messageProducer.produce(parsedFlow, msg)
else:
log.error("messageProducer has no consumers")
def fetch(self):
try:
if (self._active is False):
time.sleep(0.2)
return
if (self._pollTimeoutCount == MAX_POLLS):
log.warning('ZMQListener poll timeout reached')
self._pollTimeoutCount = 0
self._worker.close()
self._worker = self._worker_socket(self._remoteAddress)
if (self._pollTimeoutCount > 0):
self._readMessage()
return
self._send(signals.PPP_READY)
self._readMessage()
except Exception as e:
log.error('ZMQListener.fetch {e}', e=str(e))
finally:
if (self._active is False):
self._working = False
def loadAlgorithmCallbacks(self,
start,
init=None,
stop=None,
exit=None,
options=None):
try:
log.info('Initializing algorithm callbacks')
self._originalAlgorithm['start'] = start
self._originalAlgorithm['init'] = init
self._originalAlgorithm['stop'] = stop
self._originalAlgorithm['exit'] = exit
for k, v in methods.items():
methodName = k
method = v
isMandatory = method["mandatory"]
if self._originalAlgorithm[methodName] is not None:
log.info('found method {methodName}',
methodName=methodName)
else:
mandatory = "mandatory" if isMandatory else "optional"
error = 'unable to find {mandatory} method {methodName}'.format(
mandatory=mandatory, methodName=methodName)
if (isMandatory):
raise Exception(error)
log.warning(error)
# fix start if it has only one argument
if start.__code__.co_argcount == 1:
self._originalAlgorithm['start'] = lambda args, api: start(args
)
self._wrapStateless()
self._tracer = Tracer(options.tracer)
except Exception as e:
self._loadAlgorithmError = self._errorMsg(e)
log.error(e)
def _send(self, message):
try:
toBeSent = self._replyFunc(message)
self._socket.send_multipart(toBeSent, copy=False)
except Exception as e:
log.error(e)
def start(self): # pylint: disable=too-many-branches
poll_workers = zmq.Poller()
poll_workers.register(self._backend, zmq.POLLIN)
while self._active: # pylint: disable=too-many-nested-blocks
try:
socks = dict(poll_workers.poll(CYCLE_LENGTH_MS))
if socks.get(self._backend) == zmq.POLLIN:
frames = self._backend.recv_multipart() or []
if (len(frames) != 4):
log.warning("got {len} frames {frames}",
len=len(frames),
frames=frames)
continue
address, signal, consumer, result = frames # pylint: disable=unbalanced-tuple-unpacking
consumerType = self.encoding.decode(value=consumer,
plainEncode=True)
if (not consumerType in self.consumerTypes):
log.warning(
"Producer got message from unknown consumer: {consumerType}, dropping the message",
consumerType=consumerType)
continue
if (signal == signals.PPP_DONE):
sentTime = self.watingForResponse.get(address)
if (sentTime):
now = time.time()
del self.watingForResponse[address]
self.responseAccumulator(
result, consumerType,
round((now - sentTime) * 1000, 4))
else:
log.error('missing from watingForResponse:' +
str(signal))
elif (signal == signals.PPP_READY):
message = self.messageQueue.pop(consumerType)
if (message):
messageFlowPattern, header, payload, appendTime = message
flow = Flow(messageFlowPattern)
flowMsg = self.encoding.encode(flow.getRestOfFlow(
self.nodeName),
plainEncode=True)
frames = [
address, signals.PPP_MSG, flowMsg, header,
payload
]
self.watingForResponse[address] = time.time()
queueTime = round(
(time.time() - appendTime) * 1000, 4)
self.queueTimeAccumulator(consumerType, queueTime)
self._send(frames)
else:
self._send([address, signals.PPP_NO_MSG])
except Exception as e:
log.error('Error in ZMQProducer {e}', e=str(e))
self._working = False
