def _createSocket(self, host, port): connStr = 'tcp://' + host + ':' + str(port) log.info('creating socket to: {s}', s=connStr) socket = context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(connStr) return socket, connStr
def connectToWorker(self, options): socket = options.socket encoding = socket.get("encoding") self._storage = options.storage.get("mode") url = socket.get("url") if (url is not None): self._url = url else: self._url = '{protocol}://{host}:{port}'.format(**socket) self._url += '?storage={storage}&encoding={encoding}'.format( storage=self._storage, encoding=encoding) self._wsc = WebsocketClient(self._msg_queue, encoding, self._url) self._initDataAdapter(options) self.streamingManager = StreamingManager() self._hkubeApi = HKubeApi(self._wsc, self, self._dataAdapter, self._storage, self.streamingManager) self._registerToWorkerEvents() log.info('connecting to {url}', url=self._url) self._wsc.start() self.start() return [self._wsc, self]
def _createZmqPingServers(self, port): try: pingContext = zmq.Context() url_worker = "inproc://ping_workers" url_client = "tcp://*:" + str(port + 1) def createDealerRouter(url_client, url_worker, context): clients = context.socket(zmq.ROUTER) clients.bind(url_client) workers = context.socket(zmq.DEALER) workers.bind(url_worker) return (clients, workers) clients, workers = createDealerRouter(url_client=url_client, url_worker=url_worker, context=pingContext) log.info('clients: {c}, workers: {w}', c=clients.get(zmq.LAST_ENDPOINT), w=workers.get(zmq.LAST_ENDPOINT)) log.info( 'Creating {num_threads} ZMQ Ping Servers on port {port}. pid: {pid}', port=url_client, num_threads=self._num_ping_threads, pid=os.getpid()) for i in range(self._num_ping_threads): server = ZMQPingServer(pingContext, url_worker, 'Ping-Thread-' + str(i)) server.start() zmq.device(zmq.QUEUE, clients, workers) except Exception as e: log.error(e)
def start_stored_subpipeline(self, name, flowInput={}, includeResult=True): """Starts pipeline execution. starts an invocation of a sub-pipeline with input, and waits for results Args: name (string): The name of the pipeline to start. flowInput (dict): Optional flowInput for the pipeline. includeResult (bool): if True, returns the result of the pipeline execution. default: True Returns: Returns the result of the pipeline """ log.info('start_stored_subpipeline called with {name}', name=name) execId = self._generateExecId() execution = Execution(execId, includeResult, WaitForData(True)) self._executions[execId] = execution message = { "command": messages.outgoing.startStoredSubPipeline, "data": { "subPipeline": { "name": name, "flowInput": flowInput }, "subPipelineId": execId, "includeResult": includeResult } } self._wc.send(message) return self._waitForResult(execution)
def _worker_socket(self, remoteAddress): """Helper function that returns a new configured socket connected to the Paranoid Pirate queue""" identity = str(uuid.uuid4()).encode() worker = context.socket(zmq.DEALER) worker.setsockopt(zmq.IDENTITY, identity) worker.connect(remoteAddress) log.info("zmq listener connecting to {addr}", addr=remoteAddress) return worker
def __init__(self, options, dataServer=None): self._dataServer = dataServer self._storageCache = Cache(config.storage) self._encoding = Encoding(options.storage['encoding']) self._storageManager = StorageManager(options.storage) self._requestEncoding = options.storage['encoding'] self._requestTimeout = options.discovery['timeout'] self._networkTimeout = options.discovery['networkTimeout'] self._maxWorkers = min(32, (multiprocessing.cpu_count() or 1) + 4) log.info('using {workers} workers for DataAdapter', workers=self._maxWorkers)
def __init__(self, config): storageType = config["type"] encoding = config["encoding"] adapterConfig = config[storageType] adapterType = adapterTypes.get(storageType) adapter = adapterType(adapterConfig, encoding) self.hkube = TaskOutputManager(adapter, config) self.storage = BaseStorageManager(adapter) log.info('init {type} storage client with {encoding} encoding', type=storageType, encoding=encoding)
def run(self): while self._active: try: (command, data) = self.get_message() runThread = Thread(name=command + "Thread", target=self.handle, args=[command, data]) runThread.daemon = True runThread.start() except Empty: pass log.info('Exiting run loop')
def _checkQueueSize(self, event): if (self._job.isStreaming): if (self.streamingManager.messageProducer): try: log.info('Messages left in queue on {event}={queue}', event=event, queue=str( len(self.streamingManager.messageProducer. adapter.messageQueue.queue))) except Exception: log.error( 'Failed to print number of messages left in queue on {event}', event=event) else: log.info('MessageProducer already None on {event}', event=event)
def getDataSource(self, dataSource): log.info('getDataSource called') requestId = self._generateExecId() execution = Execution(requestId, False, WaitForData(True)) self._executions[requestId] = execution message = { "command": messages.outgoing.dataSourceRequest, "data": { "requestId": requestId, "dataSource": dataSource } } self._wc.send(message) return self._waitForResult(execution)
def __init__(self, port, maxMemorySize, responseAccumulator, queueTimeAccumulator, consumerTypes, encoding, nodeName): self.nodeName = nodeName self.encoding = encoding self.responseAccumulator = responseAccumulator self.queueTimeAccumulator = queueTimeAccumulator self.maxMemorySize = maxMemorySize self.port = port self.consumerTypes = consumerTypes self.messageQueue = MessageQueue(consumerTypes, self.nodeName) self._backend = context.socket(zmq.ROUTER) # ROUTER self._backend.bind("tcp://*:" + str(port)) # For workers log.info("Producer listening on tcp://*:{port}", port=port) self._active = True self._working = True self.watingForResponse = {}
def _exit(self, options): try: self._dataServer and self._dataServer.shutDown() self._wsc.shutDown() method = self._getMethod('exit') if (method is not None): method(options) self._checkQueueSize(event='exit') option = options if options is not None else dict() code = option.get('exitCode', 0) self._active = False log.info('Got exit command. Exiting with code {code}', code=code) sys.exit(code) except Exception as e: log.error('Got error during exit: {e}', e=e) # pylint: disable=protected-access os._exit(0)
def run(self): self._socket = self._context.socket(zmq.REP) self._socket.setsockopt(zmq.LINGER, 0) self._socket.connect(self._workerUrl) while self._active: try: events = self._socket.poll(timeout=1000) if (events == 0): continue message = self._socket.recv() if(message == consts.zmq.ping): self._socket.send(consts.zmq.pong) except Exception as e: log.error('socket closed: {e}', e=str(e)) break log.info('ZmqPingServer run loop exit') self.close()
def start_raw_subpipeline(self, name, nodes, flowInput, options={}, webhooks={}, includeResult=True): """Starts pipeline execution. starts an invocation of a sub-pipeline with input, nodes, options, and optionally waits for results Args: name (string): The name of the pipeline to start. nodes (array): array of node definitions (like in the pipeline descriptor) options (dict): pipeline options (like in the pipeline descriptor) webhooks (dict): webhook options (like in the pipeline descriptor) flowInput (dict): flowInput for the pipeline. includeResult (bool): if True, returns the result of the pipeline execution. default: True Returns: Returns the result of the pipeline """ log.info('start_raw_subpipeline called with {name}', name=name) execId = self._generateExecId() execution = Execution(execId, includeResult, WaitForData(True)) self._executions[execId] = execution message = { "command": messages.outgoing.startRawSubPipeline, "data": { "subPipeline": { "name": name, "nodes": nodes, "options": options, "webhooks": webhooks, "flowInput": flowInput }, "subPipelineId": execId, "includeResult": includeResult } } self._wc.send(message) return self._waitForResult(execution)
def start_algorithm(self, algorithmName, input=[], includeResult=True): """Starts algorithm execution. starts an invocation of algorithm with input, and waits for results Args: algorithmName (string): The name of the algorithm to start. input (array): Optional input for the algorithm. includeResult (bool): if True, returns the result of the algorithm execution. Returns: Returns the result of the algorithm """ log.info('start_algorithm called with {name}', name=algorithmName) execId = self._generateExecId() execution = Execution(execId, includeResult, WaitForData(True)) self._executions[execId] = execution if self._storage == 'v3': (storage, mappedInput) = self._dataAdapter.setAlgorithmStorage( self._wrapper.getCurrentJob().jobId, input) message = { "command": messages.outgoing.startAlgorithmExecution, "data": { "execId": execId, "algorithmName": algorithmName, "storageInput": mappedInput, "storage": storage, "includeResult": includeResult } } else: message = { "command": messages.outgoing.startAlgorithmExecution, "data": { "execId": execId, "algorithmName": algorithmName, "input": input, "includeResult": includeResult } } self._wc.send(message) return self._waitForResult(execution)
def invoke(self): try: log.info('tcp://{host}:{port}', host=self.request['host'], port=self.request['port']) adapter = ZMQRequest(self.request) responseFrames = adapter.invokeAdapter() results = [] for i in range(0, int(len(responseFrames) / 2)): header = responseFrames[i * 2] content = responseFrames[i * 2 + 1] decoded = self.encoding.decode(header=header, value=content) results.append((len(content), decoded)) return results except Exception as e: results = [] for _ in self.tasks: results.append((0, self._createError('unknown', str(e)))) return results finally: adapter.close()
def loadAlgorithmCallbacks(self, start, init=None, stop=None, exit=None, options=None): try: log.info('Initializing algorithm callbacks') self._originalAlgorithm['start'] = start self._originalAlgorithm['init'] = init self._originalAlgorithm['stop'] = stop self._originalAlgorithm['exit'] = exit for k, v in methods.items(): methodName = k method = v isMandatory = method["mandatory"] if self._originalAlgorithm[methodName] is not None: log.info('found method {methodName}', methodName=methodName) else: mandatory = "mandatory" if isMandatory else "optional" error = 'unable to find {mandatory} method {methodName}'.format( mandatory=mandatory, methodName=methodName) if (isMandatory): raise Exception(error) log.warning(error) # fix start if it has only one argument if start.__code__.co_argcount == 1: self._originalAlgorithm['start'] = lambda args, api: start(args ) self._wrapStateless() self._tracer = Tracer(options.tracer) except Exception as e: self._loadAlgorithmError = self._errorMsg(e) log.error(e)
def _stopAlgorithm(self, options): if (self._stopped): log.warning('Got stop command while already stopping') else: self._stopped = True try: method = self._getMethod('stop') if (method is not None): method(options) forceStop = options.get('forceStop', False) if (forceStop is True): log.info('stopping using force flag') else: log.info('stopping gracefully') if (self._job.isStreaming): if (forceStop is False): stoppingState = True def stopping(): while (stoppingState): self._sendCommand(messages.outgoing.stopping, None) time.sleep(1) stoppingThread = Thread(target=stopping) stoppingThread.start() self._hkubeApi.stopStreaming(force=forceStop) stoppingState = False stoppingThread.join() log.info( 'Joined threads send stopping and stop streaming') self._checkQueueSize(event='scale down') else: self._hkubeApi.stopStreaming(force=forceStop) if (self._runningStartThread): self._runningStartThread.join() log.info('Joined threads algorithm and stop algorithm') self._sendCommand(messages.outgoing.stopped, None) except Exception as e: self.sendError(e)
def close(self, force=True): log.info('queue size before close = {len}', len=len(self.messageQueue.queue)) while self.messageQueue.queue and not force: log.info('queue size during close = {len}', len=len(self.messageQueue.queue)) time.sleep(1) self._active = False while self._working: time.sleep(1) self._backend.close() log.info('queue size after close = {len}', len=len(self.messageQueue.queue))
def loadAlgorithm(self, options): try: cwd = os.getcwd() algOptions = options.algorithm package = algOptions["path"] entry = algOptions["entryPoint"] entryPoint = Algorunner._getEntryPoint(entry) __import__(package) os.chdir('{cwd}/{package}'.format(cwd=cwd, package=package)) log.info('loading {entry}', entry=entry) mod = importlib.import_module( '.{entryPoint}'.format(entryPoint=entryPoint), package=package) log.info('algorithm code loaded') for k, v in methods.items(): methodName = k method = v isMandatory = method["mandatory"] try: self._originalAlgorithm[methodName] = getattr( mod, methodName) # fix start if it has only one argument if methodName == 'start' and self._originalAlgorithm[ 'start'].__code__.co_argcount == 1: self._originalAlgorithm[ 'startOrig'] = self._originalAlgorithm['start'] self._originalAlgorithm[ 'start'] = lambda args, api: self._originalAlgorithm[ 'startOrig'](args) log.info('found method {methodName}', methodName=methodName) except Exception as e: mandatory = "mandatory" if isMandatory else "optional" error = 'unable to find {mandatory} method {methodName}'.format( mandatory=mandatory, methodName=methodName) if (isMandatory): raise Exception(error) log.warning(error) self._wrapStateless() self._tracer = Tracer(options.tracer) except Exception as e: self._loadAlgorithmError = self._errorMsg(e) traceback.print_exc() log.error(e)
def _connection(self): self._connected = True log.info('connected to {url}', url=self._url)
def close(self): log.info('closing zmq servers') for i in self._instances: log.info('closing zmq servers - stop') i.stop() log.info('joining zmq server threads') for i in self._instances: i.join() log.info('zmq context closing') self._context.term() log.info('zmq context closed') log.info('closed ZmqServers')
def listen(self): log.info('discovery serving on {host}:{port} with {encoding} encoding', host=self._host, port=self._port, encoding=self._encodingType) self._adapter.listen()
def _disconnect(self): if self._connected: log.info('disconnected from {url}', url=self._url) self._connected = False