def _createSocket(self, host, port):
connStr = 'tcp://' + host + ':' + str(port)
log.info('creating socket to: {s}', s=connStr)
socket = context.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(connStr)
return socket, connStr
def connectToWorker(self, options):
socket = options.socket
encoding = socket.get("encoding")
self._storage = options.storage.get("mode")
url = socket.get("url")
if (url is not None):
self._url = url
else:
self._url = '{protocol}://{host}:{port}'.format(**socket)
self._url += '?storage={storage}&encoding={encoding}'.format(
storage=self._storage, encoding=encoding)
self._wsc = WebsocketClient(self._msg_queue, encoding, self._url)
self._initDataAdapter(options)
self.streamingManager = StreamingManager()
self._hkubeApi = HKubeApi(self._wsc, self, self._dataAdapter,
self._storage, self.streamingManager)
self._registerToWorkerEvents()
log.info('connecting to {url}', url=self._url)
self._wsc.start()
self.start()
return [self._wsc, self]
def _createZmqPingServers(self, port):
try:
pingContext = zmq.Context()
url_worker = "inproc://ping_workers"
url_client = "tcp://*:" + str(port + 1)
def createDealerRouter(url_client, url_worker, context):
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
return (clients, workers)
clients, workers = createDealerRouter(url_client=url_client,
url_worker=url_worker,
context=pingContext)
log.info('clients: {c}, workers: {w}',
c=clients.get(zmq.LAST_ENDPOINT),
w=workers.get(zmq.LAST_ENDPOINT))
log.info(
'Creating {num_threads} ZMQ Ping Servers on port {port}. pid: {pid}',
port=url_client,
num_threads=self._num_ping_threads,
pid=os.getpid())
for i in range(self._num_ping_threads):
server = ZMQPingServer(pingContext, url_worker,
'Ping-Thread-' + str(i))
server.start()
zmq.device(zmq.QUEUE, clients, workers)
except Exception as e:
log.error(e)
def start_stored_subpipeline(self, name, flowInput={}, includeResult=True):
"""Starts pipeline execution.
starts an invocation of a sub-pipeline with input, and waits for results
Args:
name (string): The name of the pipeline to start.
flowInput (dict): Optional flowInput for the pipeline.
includeResult (bool): if True, returns the result of the pipeline execution.
default: True
Returns:
Returns the result of the pipeline
"""
log.info('start_stored_subpipeline called with {name}', name=name)
execId = self._generateExecId()
execution = Execution(execId, includeResult, WaitForData(True))
self._executions[execId] = execution
message = {
"command": messages.outgoing.startStoredSubPipeline,
"data": {
"subPipeline": {
"name": name,
"flowInput": flowInput
},
"subPipelineId": execId,
"includeResult": includeResult
}
}
self._wc.send(message)
return self._waitForResult(execution)
def _worker_socket(self, remoteAddress):
"""Helper function that returns a new configured socket
connected to the Paranoid Pirate queue"""
identity = str(uuid.uuid4()).encode()
worker = context.socket(zmq.DEALER)
worker.setsockopt(zmq.IDENTITY, identity)
worker.connect(remoteAddress)
log.info("zmq listener connecting to {addr}", addr=remoteAddress)
return worker
def __init__(self, options, dataServer=None):
self._dataServer = dataServer
self._storageCache = Cache(config.storage)
self._encoding = Encoding(options.storage['encoding'])
self._storageManager = StorageManager(options.storage)
self._requestEncoding = options.storage['encoding']
self._requestTimeout = options.discovery['timeout']
self._networkTimeout = options.discovery['networkTimeout']
self._maxWorkers = min(32, (multiprocessing.cpu_count() or 1) + 4)
log.info('using {workers} workers for DataAdapter', workers=self._maxWorkers)
def __init__(self, config):
storageType = config["type"]
encoding = config["encoding"]
adapterConfig = config[storageType]
adapterType = adapterTypes.get(storageType)
adapter = adapterType(adapterConfig, encoding)
self.hkube = TaskOutputManager(adapter, config)
self.storage = BaseStorageManager(adapter)
log.info('init {type} storage client with {encoding} encoding',
type=storageType,
encoding=encoding)
def run(self):
while self._active:
try:
(command, data) = self.get_message()
runThread = Thread(name=command + "Thread",
target=self.handle,
args=[command, data])
runThread.daemon = True
runThread.start()
except Empty:
pass
log.info('Exiting run loop')
def _checkQueueSize(self, event):
if (self._job.isStreaming):
if (self.streamingManager.messageProducer):
try:
log.info('Messages left in queue on {event}={queue}',
event=event,
queue=str(
len(self.streamingManager.messageProducer.
adapter.messageQueue.queue)))
except Exception:
log.error(
'Failed to print number of messages left in queue on {event}',
event=event)
else:
log.info('MessageProducer already None on {event}',
event=event)
def getDataSource(self, dataSource):
log.info('getDataSource called')
requestId = self._generateExecId()
execution = Execution(requestId, False, WaitForData(True))
self._executions[requestId] = execution
message = {
"command": messages.outgoing.dataSourceRequest,
"data": {
"requestId": requestId,
"dataSource": dataSource
}
}
self._wc.send(message)
return self._waitForResult(execution)
def __init__(self, port, maxMemorySize, responseAccumulator,
queueTimeAccumulator, consumerTypes, encoding, nodeName):
self.nodeName = nodeName
self.encoding = encoding
self.responseAccumulator = responseAccumulator
self.queueTimeAccumulator = queueTimeAccumulator
self.maxMemorySize = maxMemorySize
self.port = port
self.consumerTypes = consumerTypes
self.messageQueue = MessageQueue(consumerTypes, self.nodeName)
self._backend = context.socket(zmq.ROUTER) # ROUTER
self._backend.bind("tcp://*:" + str(port)) # For workers
log.info("Producer listening on tcp://*:{port}", port=port)
self._active = True
self._working = True
self.watingForResponse = {}
def _exit(self, options):
try:
self._dataServer and self._dataServer.shutDown()
self._wsc.shutDown()
method = self._getMethod('exit')
if (method is not None):
method(options)
self._checkQueueSize(event='exit')
option = options if options is not None else dict()
code = option.get('exitCode', 0)
self._active = False
log.info('Got exit command. Exiting with code {code}', code=code)
sys.exit(code)
except Exception as e:
log.error('Got error during exit: {e}', e=e)
# pylint: disable=protected-access
os._exit(0)
def run(self):
self._socket = self._context.socket(zmq.REP)
self._socket.setsockopt(zmq.LINGER, 0)
self._socket.connect(self._workerUrl)
while self._active:
try:
events = self._socket.poll(timeout=1000)
if (events == 0):
continue
message = self._socket.recv()
if(message == consts.zmq.ping):
self._socket.send(consts.zmq.pong)
except Exception as e:
log.error('socket closed: {e}', e=str(e))
break
log.info('ZmqPingServer run loop exit')
self.close()
def start_raw_subpipeline(self,
name,
nodes,
flowInput,
options={},
webhooks={},
includeResult=True):
"""Starts pipeline execution.
starts an invocation of a sub-pipeline with input, nodes, options, and optionally waits for results
Args:
name (string): The name of the pipeline to start.
nodes (array): array of node definitions (like in the pipeline descriptor)
options (dict): pipeline options (like in the pipeline descriptor)
webhooks (dict): webhook options (like in the pipeline descriptor)
flowInput (dict): flowInput for the pipeline.
includeResult (bool): if True, returns the result of the pipeline execution.
default: True
Returns:
Returns the result of the pipeline
"""
log.info('start_raw_subpipeline called with {name}', name=name)
execId = self._generateExecId()
execution = Execution(execId, includeResult, WaitForData(True))
self._executions[execId] = execution
message = {
"command": messages.outgoing.startRawSubPipeline,
"data": {
"subPipeline": {
"name": name,
"nodes": nodes,
"options": options,
"webhooks": webhooks,
"flowInput": flowInput
},
"subPipelineId": execId,
"includeResult": includeResult
}
}
self._wc.send(message)
return self._waitForResult(execution)
def start_algorithm(self, algorithmName, input=[], includeResult=True):
"""Starts algorithm execution.
starts an invocation of algorithm with input, and waits for results
Args:
algorithmName (string): The name of the algorithm to start.
input (array): Optional input for the algorithm.
includeResult (bool): if True, returns the result of the algorithm execution.
Returns:
Returns the result of the algorithm
"""
log.info('start_algorithm called with {name}', name=algorithmName)
execId = self._generateExecId()
execution = Execution(execId, includeResult, WaitForData(True))
self._executions[execId] = execution
if self._storage == 'v3':
(storage, mappedInput) = self._dataAdapter.setAlgorithmStorage(
self._wrapper.getCurrentJob().jobId, input)
message = {
"command": messages.outgoing.startAlgorithmExecution,
"data": {
"execId": execId,
"algorithmName": algorithmName,
"storageInput": mappedInput,
"storage": storage,
"includeResult": includeResult
}
}
else:
message = {
"command": messages.outgoing.startAlgorithmExecution,
"data": {
"execId": execId,
"algorithmName": algorithmName,
"input": input,
"includeResult": includeResult
}
}
self._wc.send(message)
return self._waitForResult(execution)
def invoke(self):
try:
log.info('tcp://{host}:{port}',
host=self.request['host'],
port=self.request['port'])
adapter = ZMQRequest(self.request)
responseFrames = adapter.invokeAdapter()
results = []
for i in range(0, int(len(responseFrames) / 2)):
header = responseFrames[i * 2]
content = responseFrames[i * 2 + 1]
decoded = self.encoding.decode(header=header, value=content)
results.append((len(content), decoded))
return results
except Exception as e:
results = []
for _ in self.tasks:
results.append((0, self._createError('unknown', str(e))))
return results
finally:
adapter.close()
def loadAlgorithmCallbacks(self,
start,
init=None,
stop=None,
exit=None,
options=None):
try:
log.info('Initializing algorithm callbacks')
self._originalAlgorithm['start'] = start
self._originalAlgorithm['init'] = init
self._originalAlgorithm['stop'] = stop
self._originalAlgorithm['exit'] = exit
for k, v in methods.items():
methodName = k
method = v
isMandatory = method["mandatory"]
if self._originalAlgorithm[methodName] is not None:
log.info('found method {methodName}',
methodName=methodName)
else:
mandatory = "mandatory" if isMandatory else "optional"
error = 'unable to find {mandatory} method {methodName}'.format(
mandatory=mandatory, methodName=methodName)
if (isMandatory):
raise Exception(error)
log.warning(error)
# fix start if it has only one argument
if start.__code__.co_argcount == 1:
self._originalAlgorithm['start'] = lambda args, api: start(args
)
self._wrapStateless()
self._tracer = Tracer(options.tracer)
except Exception as e:
self._loadAlgorithmError = self._errorMsg(e)
log.error(e)
def _stopAlgorithm(self, options):
if (self._stopped):
log.warning('Got stop command while already stopping')
else:
self._stopped = True
try:
method = self._getMethod('stop')
if (method is not None):
method(options)
forceStop = options.get('forceStop', False)
if (forceStop is True):
log.info('stopping using force flag')
else:
log.info('stopping gracefully')
if (self._job.isStreaming):
if (forceStop is False):
stoppingState = True
def stopping():
while (stoppingState):
self._sendCommand(messages.outgoing.stopping,
None)
time.sleep(1)
stoppingThread = Thread(target=stopping)
stoppingThread.start()
self._hkubeApi.stopStreaming(force=forceStop)
stoppingState = False
stoppingThread.join()
log.info(
'Joined threads send stopping and stop streaming')
self._checkQueueSize(event='scale down')
else:
self._hkubeApi.stopStreaming(force=forceStop)
if (self._runningStartThread):
self._runningStartThread.join()
log.info('Joined threads algorithm and stop algorithm')
self._sendCommand(messages.outgoing.stopped, None)
except Exception as e:
self.sendError(e)
def close(self, force=True):
log.info('queue size before close = {len}',
len=len(self.messageQueue.queue))
while self.messageQueue.queue and not force:
log.info('queue size during close = {len}',
len=len(self.messageQueue.queue))
time.sleep(1)
self._active = False
while self._working:
time.sleep(1)
self._backend.close()
log.info('queue size after close = {len}',
len=len(self.messageQueue.queue))
def loadAlgorithm(self, options):
try:
cwd = os.getcwd()
algOptions = options.algorithm
package = algOptions["path"]
entry = algOptions["entryPoint"]
entryPoint = Algorunner._getEntryPoint(entry)
__import__(package)
os.chdir('{cwd}/{package}'.format(cwd=cwd, package=package))
log.info('loading {entry}', entry=entry)
mod = importlib.import_module(
'.{entryPoint}'.format(entryPoint=entryPoint), package=package)
log.info('algorithm code loaded')
for k, v in methods.items():
methodName = k
method = v
isMandatory = method["mandatory"]
try:
self._originalAlgorithm[methodName] = getattr(
mod, methodName)
# fix start if it has only one argument
if methodName == 'start' and self._originalAlgorithm[
'start'].__code__.co_argcount == 1:
self._originalAlgorithm[
'startOrig'] = self._originalAlgorithm['start']
self._originalAlgorithm[
'start'] = lambda args, api: self._originalAlgorithm[
'startOrig'](args)
log.info('found method {methodName}',
methodName=methodName)
except Exception as e:
mandatory = "mandatory" if isMandatory else "optional"
error = 'unable to find {mandatory} method {methodName}'.format(
mandatory=mandatory, methodName=methodName)
if (isMandatory):
raise Exception(error)
log.warning(error)
self._wrapStateless()
self._tracer = Tracer(options.tracer)
except Exception as e:
self._loadAlgorithmError = self._errorMsg(e)
traceback.print_exc()
log.error(e)
def _connection(self):
self._connected = True
log.info('connected to {url}', url=self._url)
def close(self):
log.info('closing zmq servers')
for i in self._instances:
log.info('closing zmq servers - stop')
i.stop()
log.info('joining zmq server threads')
for i in self._instances:
i.join()
log.info('zmq context closing')
self._context.term()
log.info('zmq context closed')
log.info('closed ZmqServers')
def listen(self):
log.info('discovery serving on {host}:{port} with {encoding} encoding',
host=self._host,
port=self._port,
encoding=self._encodingType)
self._adapter.listen()
def _disconnect(self):
if self._connected:
log.info('disconnected from {url}', url=self._url)
self._connected = False
