Example #1
0
 def sRecvOnSubPub(self, iFlags=zmq.NOBLOCK):
     if self.oSubPubSocket is None:
         # was self.eBindListener()
         # needs lTopics: self.eConnectToSubPub(lTopics)
         pass
     assert self.oSubPubSocket, "sRecvOnSubPub: oSubPubSocket is null"
     try:
         sRetval = self.oSubPubSocket.recv(flags=iFlags)
     except zmq.ZMQError as e:
         # zmq4: iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             #? This should only occur if iFlags are zmq.NOBLOCK
             time.sleep(1.0)
         else:
             vWarn("sRecvOnSubPub: ZMQError in Recv listener: %d %s" % (
                 iError,
                 zmq.strerror(iError),
             ))
             sys.stdout.flush()
         sRetval = ""
     except Exception as e:
         vError("sRecvOnSubPub: Failed Recv listener: " + str(e))
         sys.stdout.flush()
         sRetval = ""
     return sRetval
Example #2
0
    def checkForMessage(self, socket):
        """ Check on socket activity if there is a complete ZMQ message.

          @param socket: ZMQ socket
        """
        logging.debug("Check: {0!s}".format(self.readnotifier.socket()))
        self.readnotifier.setEnabled(False)
        check = True
        try:
            while check:
                events = self.socket.get(zmq.EVENTS)
                check = events & zmq.POLLIN
                logging.debug("EVENTS: {0!s}".format(events))
                if check:
                    try:
                        msg = self.socket.recv_multipart(zmq.NOBLOCK)
                    except zmq.ZMQError as e:
                        if e.errno == zmq.EAGAIN:
                            logging.debug("state changed since poll event")
                            # state changed since poll event
                            pass
                        else:
                            logging.info("RECV Error: {0!s}".format(
                                zmq.strerror(e.errno)))
                    else:
                        logging.debug("MSG: {0!s} {1!s}".format(
                            self.readnotifier.socket(), msg))
                        self.sigMsgRecvd.emit(msg)
        except:
            logging.debug("Exception in QZMQStream::checkForMessages")
            pass
        else:
            self.readnotifier.setEnabled(True)
Example #3
0
 def eSendOnReqRep(self, sTopic, sMsg):
     assert sMsg.startswith(sTopic), \
            "eSendOnReqRep: sMsg.startswith(sTopic) failed" + sMsg +" " +sTopic
     if sTopic not in lKNOWN_TOPICS:
         sRetval = "eSendOnReqRep: eSendOnReqRep unhandled topic: " + sTopics + " " + sMess
         vError(sRetval)
         return sRetval
     assert self.oReqRepSocket, "eSendOnReqRep: oReqRepSocket is null"
     try:
         sRetval = self.oReqRepSocket.send(sMsg)
     except zmq.ZMQError as e:
         # iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             time.sleep(1.0)
             sRetval = ""
         else:
             sRetval = zmq.strerror(iError)
             vWarn("eSendOnReqRep: ZMQError: %d %s" % (
                 iError,
                 sRetval,
             ))
             sys.stdout.flush()
     except Exception as e:
         vError("eSendOnReqRep: Failed: " + str(e))
         sys.stdout.flush()
         sRetval = str(e)
     return sRetval
Example #4
0
    async def fetch( self, query: str, method: str='GET', headers: Mapping[str,str]={}, 
                     data: Any=None, timeout: int=5) -> Any:
        """ Send a request message to the worker
        """
        # Send request
        request = pickle.dumps(RequestMessage(query,headers=headers,method=method,data=data),-1)
        correlation_id = uuid.uuid1().bytes
        assert correlation_id not in self._handlers
        try:
            await self._socket.send_multipart([correlation_id, request], flags=zmq.DONTWAIT)
        except zmq.ZMQError as err:
            LOGGER.error("%s (%s)", zmq.strerror(err.errno), err.errno)
            raise RequestGatewayError()
        
        # Create response handler and register it
        handler = AsyncResponseHandler(correlation_id)
        self._handlers[correlation_id] = handler
        # Run poller if needed
        if not self._polling:
            asyncio.ensure_future(self._poll())

        # Wait for response
        try:
            return await handler._get(timeout)
        except Exception:
            self._handlers.pop(correlation_id,None)
            raise
Example #5
0
 async def _poll(self) -> None:
     """ Handle incoming messages
     """
     self._polling = True
     while self._handlers:
         try:
             correlation_id, data, *rest = await self._socket.recv_multipart(
             )
             # Get if there is a future pending for that message
             try:
                 handler = self._handlers[correlation_id]
                 if rest and data == b'ERR':
                     handler._set_exception(RequestProxyError(rest[0]))
                 else:
                     handler._set_result(data)
                 # Remove handlers from the heap if we are done
                 if handler._done():
                     self._handlers.pop(correlation_id, None)
             except KeyError:
                 LOGGER.warning(
                     "%s: No pending future found for message %s",
                     self.identity, correlation_id)
         except zmq.ZMQError as err:
             LOGGER.error("%s error:  zmq error: %s (%s)", self.identity,
                          zmq.strerror(err.errno), err.errno)
         except Exception as err:
             LOGGER.error("%s exception %s\n%s", self.identity, err,
                          traceback.format_exc())
     self._polling = False
Example #6
0
    async def _run_async(self) -> None:
        """ Run supervisor
        """
        loop = asyncio.get_event_loop()

        def kill(pid:int) -> None:
            LOGGER.critical("Killing stalled process %s", pid)
            del self._busy[pid]
            self._killfunc(pid)

        self._stopped = False

        while not self._stopped:
            try:
                pid, notif = await self._sock.recv_multipart()
                pid = int(pid)
                if notif == b'BUSY':
                    self._busy[pid] = loop.call_later(self._timeout,kill,pid)
                elif notif == b'DONE':
                    try:
                        self._busy.pop(pid).cancel()
                    except KeyError:
                        pass
            except zmq.ZMQError as err:
                if err.errno != zmq.EAGAIN:
                    LOGGER.error("%s\n%s", zmq.strerror(err.errno), traceback.format_exc())
            except asyncio.CancelledError:
                self._stopped = True
            except Exception:
                LOGGER.critical("%s", traceback.format_exc())
Example #7
0
    async def _apply_async(self, request: bytes, timeout: int = 5) -> Any:
        """ Run job asynchronously
        """
        if len(self._handlers) > self._maxqueue:
            raise MaxRequestExceeded()

        # Wait for available worker
        worker_id = await self._get_worker(timeout)

        # Send request
        correlation_id = uuid.uuid1().bytes
        try:
            # Pick available worker
            await self._socket.send_multipart(
                [worker_id, correlation_id, request], flags=zmq.DONTWAIT)
        except zmq.ZMQError as err:
            LOGGER.error("%s (%s)", zmq.strerror(err.errno), err.errno)
            raise RequestGatewayError()

        handler = self._loop.create_future()

        try:
            self._handlers[correlation_id] = handler
            # Wait for response
            return await asyncio.wait_for(handler, timeout)
        finally:
            # Remove the handler
            self._handlers.pop(correlation_id, None)
Example #8
0
 def _send(self, data: bytes) -> None:
     try:
         self._sock.send_multipart([str(self._pid).encode(), data],
                                   flags=zmq.DONTWAIT)
     except zmq.ZMQError as err:
         if err.errno != zmq.EAGAIN:
             LOGGER.error("%s (%s)", zmq.strerror(err.errno), err.errno)
Example #9
0
    async def _poll(self) -> None:
        """ Handle incoming messages
        """
        cancelled = False
        while not cancelled:
            try:
                worker_id, *rest = await self._socket.recv_multipart()
                if rest[0] == WORKER_READY:
                    # Worker is available on new connection
                    # Mark worker as available
                    self._put_worker(worker_id)
                    continue

                msgid, (success, response) = (rest[0], pickle.loads(rest[1]))
                LOGGER.debug("Receveid %s, success %s", msgid, success)
                # Get if there is a future pending for that message
                try:
                    handler = self._handlers.pop(msgid)
                    if not success:
                        response = RequestBackendError(response)
                        handler.set_exception(response)
                    else:
                        handler.set_result(response)
                except KeyError:
                    LOGGER.warning(
                        "%s: No pending future found for message %s",
                        self.identity, correlation_id)
            except zmq.ZMQError as err:
                LOGGER.error("%s error:  zmq error: %s (%s)", self.identity,
                             zmq.strerror(err.errno), err.errno)
            except asyncio.CancelledError:
                LOGGER.debug("polling stopped")
                cancelled = True
Example #10
0
 def eSendOnReqRep(self, sTopic, sMsg):
     assert sMsg.startswith(sTopic), \
            "eSendOnReqRep: sMsg.startswith(sTopic) failed" + sMsg +" " +sTopic
     if sTopic not in lKNOWN_TOPICS:
         sRetval = "eSendOnReqRep: eSendOnReqRep unhandled topic: " +sTopics +" " +sMess
         vError(sRetval)
         return sRetval
     assert self.oReqRepSocket, "eSendOnReqRep: oReqRepSocket is null"
     try:
         sRetval = self.oReqRepSocket.send(sMsg)
     except zmq.ZMQError as e:
         # iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             time.sleep(1.0)
             sRetval = ""
         else:
             sRetval = zmq.strerror(iError)
             vWarn("eSendOnReqRep: ZMQError: %d %s" % (
                 iError, sRetval,))
             sys.stdout.flush()
     except Exception as e:
         vError("eSendOnReqRep: Failed: " +str(e))
         sys.stdout.flush()
         sRetval = str(e)
     return sRetval
Example #11
0
    def checkForMessage(self, socket):
        """ Check on socket activity if there is a complete ZMQ message.

          @param socket: ZMQ socket
        """
        logging.debug( "Check: {0!s}".format(self.readnotifier.socket()))
        self.readnotifier.setEnabled(False)
        check = True
        try:
            while check:
                events = self.socket.get(zmq.EVENTS)
                check = events & zmq.POLLIN
                logging.debug( "EVENTS: {0!s}".format(events))
                if check:
                    try:
                        msg = self.socket.recv_multipart(zmq.NOBLOCK)
                    except zmq.ZMQError as e:
                        if e.errno == zmq.EAGAIN:
                            # state changed since poll event
                            pass
                        else:
                            logging.info( "RECV Error: {0!s}".format(zmq.strerror(e.errno)))
                    else:
                        logging.debug( "MSG: {0!s} {1!s}".format(self.readnotifier.socket(), msg))
                        self.sigMsgRecvd.emit(msg)
        except:
            pass
        else:
            self.readnotifier.setEnabled(True)
Example #12
0
    async def _run_async(self) -> Awaitable[None]:
        """ Run supervisor
        """
        loop = asyncio.get_event_loop()

        def kill(pid: int) -> None:
            del self._busy[pid]
            try:
                os.kill(pid, signal.SIGKILL)
                LOGGER.critical("Killed stalled process %s", pid)
            except ProcessLookupError:
                # Process was already terminated/crashed
                pass

        self._stopped = False

        while not self._stopped:
            try:
                pid, notif = await self._sock.recv_pyobj()
                if notif == b'BUSY':
                    self._busy[pid] = loop.call_later(self._timeout, kill, pid)
                elif notif == b'DONE':
                    try:
                        self._busy.pop(pid).cancel()
                    except KeyError:
                        pass
            except zmq.ZMQError as err:
                if err.errno != zmq.EAGAIN:
                    LOGGER.error("%s\n%s", zmq.strerror(err.errno),
                                 traceback.format_exc())
            except asyncio.CancelledError:
                raise
            except Exception:
                LOGGER.critical("%s", traceback.format_exc())
                raise
Example #13
0
    def run(self):

        sys.stdout.write("Starting Zmq ListenerThread listening to: " +
                         repr(self.lTopics) + "\n")
        if self.oSubPubSocket is None:
            try:
                self.eConnectToSub(self.lTopics)
            except Exception as e:
                sys.stdout.write(
                    "ERROR: starting listener thread eConnectToSub " + str(e))
                sys.stderr.write(traceback.format_exc(10) + "\n")
                return

        self._running.set()
        while self._running.is_set():
            try:
                #? 0 is blocking
                sString = self.sRecvOnSubPub(iFlags=0)
                if sString == "": continue
                self.vCallbackOnListener(sString)
            except zmq.ZMQError as e:
                # zmq4: iError = zmq.zmq_errno()
                iError = e.errno
                if iError == zmq.EAGAIN:
                    #? This should only occur if iFlags are zmq.NOBLOCK
                    time.sleep(1.0)
                else:
                    sys.stdout.write(
                        "WARN: run ZMQError in Recv listener: %d %s" % (
                            iError,
                            zmq.strerror(iError),
                        ))
                    sys.stdout.flush()
                sRetval = ""
            except (KeyboardInterrupt, ) as e:
                # Basic.Cancel
                sys.stdout.write("DEBUG: stopping listener thread " + str(e) +
                                 "\n")
                self.stop()
            except Exception as e:
                sys.stdout.write(
                    "WARN: unhandled error - stopping listener thread " +
                    str(e) + "\n")
                #? raise
                self.stop()
        try:
            if self.oSubPubSocket:
                self.oSubPubSocket.setsockopt(zmq.LINGER, 0)
                time.sleep(0.1)
                self.oSubPubSocket.close()
                self.oSubPubSocket = None
        except Exception as e:
            sys.stdout.write("WARN: error closing listener thread connection" +
                             str(e) + "\n")
            sys.stdout.flush()
Example #14
0
    def _handle_on_send(self):
        '''
        Hadles the pending message to be sent across this socket.
        '''
        msg = self._send_queue.popleft()
        try:
            if self._on_send_callback is not None:
                self._on_send_callback(msg)

            self._socket.send_multipart(msg)
        except zmq.ZMQError as e:
            log.exception("Send error: %s", zmq.strerror(e.errno))
Example #15
0
    def _handle_on_recv(self):
        '''
        Handles the pending received messages on this socket.
        '''
        try:
            msgs = self._socket.recv_multipart(zmq.NOBLOCK)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # state changed since poll event
                pass
            else:
                log.exception("Recv error: %s", zmq.strerror(e.errno))

        if self._on_recv_callback is not None:
            self._on_recv_callback(msgs)
Example #16
0
 def _handle_recv(self):
     """Handle a recv event."""
     if self._flushed:
         return
     try:
         msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy)
     except zmq.ZMQError as e:
         if e.errno == zmq.EAGAIN:
             # state changed since poll event
             pass
         else:
             logging.error("RECV Error: %s" % zmq.strerror(e.errno))
     else:
         if self._recv_callback:
             callback = self._recv_callback
             # self._recv_callback = None
             self._run_callback(callback, msg)
    def run(self):

        sys.stdout.write("Starting Zmq ListenerThread listening to: " + repr(self.lTopics) +"\n")
        if self.oSubPubSocket is None:
            try:
                self.eConnectToSub(self.lTopics)
            except Exception as e:
                sys.stdout.write("ERROR: starting listener thread eConnectToSub " +str(e))
                sys.stderr.write(traceback.format_exc(10) +"\n")
                return

        self._running.set()
        while self._running.is_set():
            try:
                #? 0 is blocking
                sString = self.sRecvOnSubPub(iFlags=0)
                if sString == "": continue
                self.vCallbackOnListener(sString)
            except zmq.ZMQError as e:
                # zmq4: iError = zmq.zmq_errno()
                iError = e.errno
                if iError == zmq.EAGAIN:
                    #? This should only occur if iFlags are zmq.NOBLOCK
                    time.sleep(1.0)
                else:
                    sys.stdout.write("WARN: run ZMQError in Recv listener: %d %s" % (
                        iError, zmq.strerror(iError),))
                    sys.stdout.flush()
                sRetval = ""
            except (KeyboardInterrupt,) as e:
                # Basic.Cancel
                sys.stdout.write("DEBUG: stopping listener thread " +str(e) +"\n")
                self.stop()
            except Exception as e:
                sys.stdout.write("WARN: unhandled error - stopping listener thread " +str(e) +"\n")
                #? raise
                self.stop()
        try:
            if self.oSubPubSocket:
                self.oSubPubSocket.setsockopt(zmq.LINGER, 0)
                time.sleep(0.1)
                self.oSubPubSocket.close()
                self.oSubPubSocket = None
        except Exception as e:
            sys.stdout.write("WARN: error closing listener thread connection" +str(e) +"\n")
            sys.stdout.flush()
Example #18
0
 def _handle_recv(self):
     """Handle a recv event."""
     if self._flushed:
         return
     try:
         msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy)
     except zmq.ZMQError as e:
         if e.errno == zmq.EAGAIN:
             # state changed since poll event
             pass
         else:
             gen_log.error("RECV Error: %s"%zmq.strerror(e.errno))
     else:
         if self._recv_callback:
             callback = self._recv_callback
             # self._recv_callback = None
             self._run_callback(callback, msg)
Example #19
0
    async def _poll(self) -> None:
        """ Handle incoming messages
        """
        cancelled = False
        while not cancelled:
            try:
                worker_id, *rest = await self._socket.recv_multipart()
                if rest[0] == WORKER_READY:
                    # Worker is available on new connection
                    # Mark worker as available
                    self._put_worker(worker_id)
                    continue
                if rest[0] == WORKER_DONE:
                    # Worker is gone because of a restart
                    # Remove worker from list
                    self._remove_worker(worker_id)
                    continue

                msgid = rest[0]
                # Get if there is a future pending for that message
                handler = self._handlers.pop(msgid, None)
                if handler is not None:
                    try:
                        success, response = pickle.loads(rest[1])
                    except Exception as exc:
                        LOGGER.error(traceback.format_exc())
                        handler.set_exception(exc)
                        continue
                    LOGGER.debug("Receveid %s, success %s", msgid, success)
                    if not success:
                        response = RequestBackendError(response)
                        handler.set_exception(response)
                    else:
                        handler.set_result(response)
                else:
                    LOGGER.warning("No pending future found for message %s",
                                   msgid)
            except zmq.ZMQError as err:
                LOGGER.error("zmq error: %s (%s)", zmq.strerror(err.errno),
                             err.errno)
            except asyncio.CancelledError:
                LOGGER.debug("polling stopped")
                cancelled = True
            except Exception:
                LOGGER.error("Polling error\n%s", traceback.format_exc())
Example #20
0
    def _on_read(self):
        if not self._poll.active:
            return

        try:
            msg = self.socket.recv_multipart(zmq.NOBLOCK,
                    copy=self._read_copy, track=self._read_track)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # state changed since poll event
                return
            else:
                logging.error("RECV Error: %s" % zmq.strerror(e.errno))
                self._read_cb(self, None, e.errno)
        else:
            self._read_cb(self, msg, None)

        self._prepare()
Example #21
0
 def sRecvOnReqRep(self, iFlag=zmq.NOBLOCK):
     assert self.oReqRepSocket, "sRecvOnReqRep: oReqRepSocket is null"
     try:
         sRetval = self.oReqRepSocket.recv(flags=iFlag)
     except zmq.ZMQError as e:
         # iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             time.sleep(1.0)
         else:
             vWarn("sRecvOnReqRep: ZMQError in Recv listener: %d %s" % (
                 iError, zmq.strerror(iError),))
             sys.stdout.flush()
         sRetval = ""
     except Exception as e:
         vError("sRecvOnReqRep: Failed Recv listener: " +str(e))
         sys.stdout.flush()
         sRetval = ""
     return sRetval
Example #22
0
 def sRecvOnReqRep(self, iFlag=zmq.NOBLOCK):
     assert self.oReqRepSocket, "sRecvOnReqRep: oReqRepSocket is null"
     try:
         sRetval = self.oReqRepSocket.recv(flags=iFlag)
     except zmq.ZMQError as e:
         # iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             time.sleep(1.0)
         else:
             vWarn("sRecvOnReqRep: ZMQError in Recv listener: %d %s" % (
                 iError,
                 zmq.strerror(iError),
             ))
             sys.stdout.flush()
         sRetval = ""
     except Exception as e:
         vError("sRecvOnReqRep: Failed Recv listener: " + str(e))
         sys.stdout.flush()
         sRetval = ""
     return sRetval
Example #23
0
 def sRecvOnSubPub(self, iFlags=zmq.NOBLOCK):
     if self.oSubPubSocket is None:
         # was self.eBindListener()
         # needs lTopics: self.eConnectToSubPub(lTopics)
         pass
     assert self.oSubPubSocket, "sRecvOnSubPub: oSubPubSocket is null"
     try:
         sRetval = self.oSubPubSocket.recv(flags=iFlags)
     except zmq.ZMQError as e:
         # zmq4: iError = zmq.zmq_errno()
         iError = e.errno
         if iError == zmq.EAGAIN:
             #? This should only occur if iFlags are zmq.NOBLOCK
             time.sleep(1.0)
         else:
             vWarn("sRecvOnSubPub: ZMQError in Recv listener: %d %s" % (
                 iError, zmq.strerror(iError),))
             sys.stdout.flush()
         sRetval = ""
     except Exception as e:
         vError("sRecvOnSubPub: Failed Recv listener: " +str(e))
         sys.stdout.flush()
         sRetval = ""
     return sRetval
Example #24
0
    async def _run_async(self) -> Awaitable[None]:
        """ Run supervisor
        """
        self._stopped = False

        while not self._stopped:
            try:
                pid, (key, modified_time,
                      state) = await self._sock.recv_pyobj()
                LOGGER.debug(
                    "*** CACHE OBSERVER: Received update %s for key %s from pid %s",
                    state, key, pid)

                # Check if an entry exists already
                entry = self._last_updates.get(key)
                if entry:
                    do_notify = entry.modified_time < modified_time
                else:
                    # Do trigger in all cases (UPDATED, INSERTED)
                    do_notify = True

                # Update entry
                self._last_updates[key] = _CacheUpdate(modified_time, state)

                if do_notify:
                    self.notify_observers(key, modified_time, state)

            except zmq.ZMQError as err:
                if err.errno != zmq.EAGAIN:
                    LOGGER.error("%s\n%s", zmq.strerror(err.errno),
                                 traceback.format_exc())
            except asyncio.CancelledError:
                raise
            except Exception:
                LOGGER.critical("%s", traceback.format_exc())
                raise
Example #25
0
 def _send(self, data: Any) -> None:
     try:
         self._sock.send_pyobj((self._pid, data), flags=zmq.DONTWAIT)
     except zmq.ZMQError as err:
         if err.errno != zmq.EAGAIN:
             LOGGER.error("%s (%s)", zmq.strerror(err.errno), err.errno)
Example #26
0
 def test_zmqerror(self):
     for errno in range(10):
         e = ZMQError(errno)
         self.assertEqual(e.errno, errno)
         self.assertEqual(str(e), strerror(errno))
Example #27
0
 def test_strerror(self):
     """test that strerror gets the right type."""
     for i in range(10):
         e = strerror(i)
         self.assertTrue(isinstance(e, str))
Example #28
0
 def test_zmqerror(self):
     for errno in range(10):
         e = ZMQError(errno)
         self.assertEqual(e.errno, errno)
         self.assertEqual(str(e), strerror(errno))
Example #29
0
    def subscribe_run(self):
        """subscribe run: start a thread to listen for messages,
        """
        self.dhelp['run'] = __doc__
        assert 'sOnlineRouting' in self.ocmd2.oCurrentSubTarget.keys(), \
               "ERROR: " +'sOnlineRouting' + " not in " +repr(self.ocmd2.oCurrentSubTarget.keys())
        assert self.ocmd2.oCurrentSubTarget['sOnlineRouting']
        sOnlineRouting = self.ocmd2.oCurrentSubTarget['sOnlineRouting']
        sChartId = self.ocmd2.sDefaultChart
        if sOnlineRouting == 'RabbitMQ':
            try:
                # PikaListenerThread needs PikaMixin
                import OpenTrader.PikaListenerThread as ListenerThread
            except ImportError as e:
                self.vError("Cant import PikaListenerThread: add the MQL4/Python directory to PYTHONPATH? " +str(e))
                self.vError("sys.path is: " +repr(sys.path))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
            sQueueName = self.ocmd2.oCurrentSubTarget['sQueueName']
        elif sOnlineRouting == 'ZeroMQ':
            try:
                import OpenTrader.ZmqListenerThread as ListenerThread
            except ImportError as e:
                self.vError("Cant import ZmqListenerThread: add the MQL4/Python directory to PYTHONPATH? " +str(e))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        else:
            raise RuntimeError("sOnlineRouting value in %s section of Cmd2.ini not supported" % (
                sOnlineRouting,))

        if self.ocmd2.oListenerThread is not None:
            self.vWarn("ListenerThread already listening to: " + repr(self.ocmd2.oListenerThread.lTopics))
            return
        #? Off by one
        lArgs = self.lArgs
        if len(lArgs) > 1:
            self.ocmd2.lTopics = lArgs[1:]
        else:
            self.ocmd2.lTopics = ['']
        if sOnlineRouting == 'RabbitMQ':
            from pika import exceptions
            try:
                dConfig = self.ocmd2.oConfig['RabbitMQ']
                assert 'sQueueName' in dConfig, \
                       "ERROR: sQueueName not in dConfig"
                self.ocmd2.oListenerThread = ListenerThread.PikaListenerThread(sChartId,
                                                                         self.ocmd2.lTopics,
                                                                         **dConfig)
                self.ocmd2.oListenerThread.start()
            except exceptions.AMQPConnectionError as e:
                self.vError("Is the RabbitMQ server running?\n" +str(e))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        elif sOnlineRouting == 'ZeroMQ':
            import zmq
            try:
                dConfig = self.ocmd2.oConfig['ZeroMQ']
                self.ocmd2.oListenerThread = ListenerThread.ZmqListenerThread(sChartId,
                                                                         self.ocmd2.lTopics,
                                                                         **dConfig)
                self.ocmd2.oListenerThread.start()
            except zmq.ZMQError as e:
                # zmq4: iError = zmq.zmq_errno()
                iError = e.errno
                self.vWarn("run ZMQError in oListenerThread.start : %d %s" % (
                    iError, zmq.strerror(iError),))
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        return
Example #30
0
    def do_subscribe(self, oArgs, oOpts=None):
        __doc__ = sSUB__doc__
        _lCmds = ['get', 'set', 'config', 'topics', 'run', 'thread'', hide', 'show', 'pprint']

        if not oArgs:
            self.vOutput("Commands to subscribe (and arguments) are required\n")
            return

        lArgs = oArgs.split()
        sDo = lArgs[0]

        # set the target for subscribe, or to see the current target
        if self.oCurrentSubTarget is None:
            assert self.oConfig['default']['lOnlineTargets'], \
                   "ERROR: empty self.oConfig['default']['lOnlineTargets']"

        if sDo == 'get' or (sDo == 'set' and len(lArgs) == 1):
            if self.oCurrentSubTarget is None:
                l = self.oConfig['default']['lOnlineTargets']
                for sElt in l:
                    assert sElt in self.oConfig.keys(), \
                           "ERROR: ini section not found: " + sCurrentSubTarget
                if len(l) > 1:
                    self.vOutput("The subscribe online targets available are: " +repr(l))
                    return
                sCurrentSubTarget = l[0]
                self.oCurrentSubTarget = self.oConfig[sCurrentSubTarget]
                self.oCurrentSubTarget.name = sCurrentSubTarget
            else:
                sCurrentSubTarget = self.oCurrentSubTarget.name
            self.vOutput("The current subscribe online target is: " +sCurrentSubTarget)
            return

        # configure the current target for subscribe: [KEY [VAL]]
        if sDo == 'config':
            # configure the current target for subscribe:  [KEY [VAL]]
            if self.oCurrentSubTarget is None:
                assert self.oConfig['default']['lOnlineTargets'], \
                       "ERROR: empty self.oConfig['default']['lOnlineTargets']"
                l = self.oConfig['default']['lOnlineTargets']
                self.vError("Use \"sub set\" to set the current target to one of: " + repr(l))
                return
            self.vConfigOp(lArgs, self.oCurrentSubTarget)
            return

        # shows topics subscribed to.
        if sDo == 'topics':
            if self.oListenerThread:
                self.vOutput("oListenerThread.lTopics: " + repr(self.G(self.oListenerThread.lTopics)))
            else:
                self.vOutput("Default lTopics: " + repr(self.G(self.lTopics)))
            return

        if sDo == 'set':
            assert len(lArgs) > 1, \
                   "ERROR: " +"Commands to subscribe (and arguments) are required"
            sTarget = lArgs[1]
            assert sTarget in self.oConfig['default']['lOnlineTargets'], \
                  "ERROR: " +sTarget +" not in " +repr(self.oConfig['default']['lOnlineTargets'])
            assert sTarget in self.oConfig.keys(), \
                   "ERROR: " +sTarget +" not in " +repr(self.oConfig.keys())
            self.oCurrentSubTarget = self.oConfig[sTarget]
            self.oCurrentSubTarget.name = sTarget
            self.vOutput("Set target to: " + repr(self.G(sTarget)))
            return

        # thread info       - info on the thread listening for messages.
        # thread stop       - stop a thread listening for messages.
        if sDo == 'thread':
            _lCmds = ['info', 'stop', 'enumerate']
            assert len(lArgs) > 1, "ERROR: sub thread " +str(_lCmds)
            assert lArgs[1] in _lCmds, "ERROR: " +sDo +" " +str(_lCmds)
            sSubCmd = lArgs[1]

            oT = self.oListenerThread
            if sSubCmd == 'enumerate':
                self.vOutput("Threads %r" % (threading.enumerate(),))
                return
            if sSubCmd == 'info':
                if not oT:
                    self.vOutput("Listener Thread not started")
                else:
                    # len(lArgs) > 2
                    lNames = [x.name for x in threading.enumerate()]
                    if oT.name not in lNames:
                        self.vWarn("Listener Thread DIED - you must \"sub run\": " + repr(lNames))
                        self.oListenerThread = None
                    else:
                        self.vOutput("Listener Thread %r" % (oT.name,))
                return
            if sSubCmd == 'stop':
                if not self.oListenerThread:
                    self.vWarn("ListenerThread not already started")
                    return
                self.pfeedback("oListenerThread.stop()")
                self.oListenerThread.stop()
                self.oListenerThread.join()
                self.oListenerThread = None
                return
            self.vError("Unrecognized subscribe thread command: " + str(lArgs) +'\n' +__doc__)
            return

        # stop seeing TOPIC messages (e.g. tick - not a pattern)
        if sDo == 'hide':
            if not self.oListenerThread:
                self.vWarn("ListenerThread not already started")
                return
            if len(lArgs) == 1:
                self.oListenerThread.vHide()
            else:
                for sElt in lArgs[1:]:
                    self.oListenerThread.vHide(sElt)
            return

        # list the message topics that are being hidden
        # start seeing TOPIC messages (e.g. tick - not a pattern)
        if sDo == 'show':
            if not self.oListenerThread:
                self.vWarn("ListenerThread not already started")
                return
            if len(lArgs) == 1:
                self.oListenerThread.vShow()
            else:
                for sElt in lArgs[1:]:
                    self.oListenerThread.vShow(sElt)
            return

        # seeing TOPIC messages with pretty-printing,
        # with 0 - off, 1 - on, no argument - current value
        if sDo == 'pprint':
            if not self.oListenerThread:
                self.vWarn("ListenerThread not already started")
                return
            if len(lArgs) == 1:
                self.oListenerThread.vPprint('get')
            else:
                self.oListenerThread.vPprint('set', bool(lArgs[1]))
            return

        if self.oCurrentSubTarget is None:
            self.vError("Use \"sub set\" to set the current target")
            return

        assert 'sOnlineRouting' in self.oCurrentSubTarget.keys(), \
               "ERROR: " +sOnlineRouting + " not in " +repr(self.oCurrentSubTarget.keys())
        assert self.oCurrentSubTarget['sOnlineRouting']
        sOnlineRouting = self.oCurrentSubTarget['sOnlineRouting']
        sChartId = self.sDefaultChart
        if sOnlineRouting == 'RabbitMQ':
            try:
                # PikaListenerThread needs PikaMixin
                import PikaListenerThread as ListenerThread
            except ImportError as e:
                self.vError("Cant import PikaListenerThread: add the MQL4/Python directory to PYTHONPATH? " +str(e))
                self.vError("sys.path is: " +repr(sys.path))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
            sQueueName = self.oCurrentSubTarget['sQueueName']
        elif sOnlineRouting == 'ZeroMQ':
            try:
                import ZmqListenerThread as ListenerThread
            except ImportError as e:
                self.vError("Cant import ZmqListenerThread: add the MQL4/Python directory to PYTHONPATH? " +str(e))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        else:
            raise RuntimeError("sOnlineRouting value in %s section of Cmd2.ini not supported" % (
                        sOnlineRouting,))

        # start a thread to listen for messages,
        if sDo == 'run':

            if self.oListenerThread is not None:
                self.vWarn("ListenerThread already listening to: " + repr(self.oListenerThread.lTopics))
                return
            if len(lArgs) > 1:
                self.lTopics = lArgs[1:]
            else:
                self.lTopics = ['']
            if sOnlineRouting == 'RabbitMQ':
                from pika import exceptions
                try:
                    dConfig = self.oConfig['RabbitMQ']
                    assert 'sQueueName' in dConfig, \
                           "ERROR: sQueueName not in dConfig"
                    self.oListenerThread = ListenerThread.PikaListenerThread(sChartId,
                                                                             self.lTopics,
                                                                             **dConfig)
                    self.oListenerThread.start()
                except exceptions.AMQPConnectionError as e:
                    self.vError("Is the RabbitMQ server running?\n" +str(e))
                    raise
                except Exception as e:
                    self.vError(traceback.format_exc(10))
                    raise
            elif sOnlineRouting == 'ZeroMQ':
                import zmq
                try:
                    dConfig = self.oConfig['ZeroMQ']
                    self.oListenerThread = ListenerThread.ZmqListenerThread(sChartId,
                                                                             self.lTopics,
                                                                             **dConfig)
                    self.oListenerThread.start()
                except zmq.ZMQError as e:
                    # zmq4: iError = zmq.zmq_errno()
                    iError = e.errno
                    self.vWarn("run ZMQError in oListenerThread.start : %d %s" % (
                        iError, zmq.strerror(iError),))
                except Exception as e:
                    self.vError(traceback.format_exc(10))
                    raise
            return

        self.vError("Unrecognized subscribe command: " + str(oArgs) +'\n' +__doc__)
Example #31
0
def run_worker(address: str,
               handler_factory: HandlerFactory,
               identity: Optional[bytes] = None,
               broadcastaddr: Optional[str] = None,
               maxcycles: Optional[int] = None) -> None:
    """ Enter the message loop
    """
    ctx = zmq.Context.instance()

    sock = dealer_socket(ctx, address, identity)
    if broadcastaddr:
        sub = broadcast_socket(ctx, broadcastaddr)

    # Initialize supervisor client
    supervisor = SupervisorClient()

    def get():
        client_id, corr_id, request = sock.recv_multipart()
        LOGGER.debug("RCV %s: %s", client_id, corr_id)
        return client_id, corr_id, pickle.loads(request)

    try:
        LOGGER.info("Starting ZMQ worker loop")
        completed = 0
        while maxcycles is None or (maxcycles and completed < maxcycles):
            sock.send(WORKER_READY)
            try:
                client_id, corr_id, request = get()
                supervisor.notify_busy()
                handler = handler_factory(sock, client_id, corr_id, request)
                handler.handle_message()
                completed += 1
            except zmq.error.Again:
                pass
            except zmq.ZMQError as err:
                LOGGER.error("Worker Error %d: %s", err.errno,
                             zmq.strerror(err.errno))
            except Exception as exc:
                LOGGER.error("Worker Error %s\n%s", exc,
                             traceback.format_exc())
                if not handler.header_written:
                    handler.status_code = 500
                    handler.send(bytes(
                        "Worker internal error".encode('ascii')))
                    # Got error 500, do not presume worker state
                    break
            finally:
                supervisor.notify_done()

            # Handle broadcast restart
            try:
                if broadcastaddr and sub.recv(flags=zmq.NOBLOCK) == b'RESTART':
                    # There is no really way to restart
                    # so exit and let the framework restart a new worker
                    LOGGER.info("Exiting on RESTART notification")
                    break
            except zmq.error.Again:
                pass
    except (KeyboardInterrupt, SystemExit):
        pass

    if broadcastaddr:
        sub.close()
    sock.close()
    LOGGER.info("Terminating Worker")
Example #32
0
def run_worker(address: str,
               handler_factory: Callable[
                   [zmq.Socket, bytes, bytes, HTTPRequest], RequestHandler],
               identity: bytes = None,
               broadcastaddr: str = None) -> None:
    """ Enter the message loop
    """
    ctx = zmq.Context.instance()

    LOGGER.info("Connecting to %s", address)
    sock = ctx.socket(zmq.DEALER)
    sock.setsockopt(zmq.LINGER, 500)  # Needed for socket no to wait on close
    sock.setsockopt(zmq.SNDHWM, 1)  # Max 1 item on send queue
    sock.setsockopt(zmq.IMMEDIATE,
                    1)  # Do not queue if no peer, will block on send
    sock.setsockopt(zmq.RCVTIMEO, 1000)  # Heartbeat
    sock.identity = identity or uuid.uuid1().bytes
    LOGGER.info("Identity set to %s", sock.identity)
    sock.connect(address)

    if broadcastaddr:
        LOGGER.info("Enabling broadcast notification")
        ctx = zmq.Context.instance()
        sub = ctx.socket(zmq.SUB)
        sub.setsockopt(zmq.LINGER,
                       500)  # Needed for socket no to wait on close
        sub.setsockopt(zmq.SUBSCRIBE, b'RESTART')
        sub.connect(broadcastaddr)

    # Initialize supervisor client
    supervisor = SupervisorClient()

    try:
        LOGGER.info("Starting ZMQ worker loop")
        while True:
            try:
                sock.send(WORKER_READY)
                client_id, corr_id, request = sock.recv_multipart()
                LOGGER.debug("RCV %s: %s", client_id, corr_id)
                request = pickle.loads(request)

                supervisor.notify_busy()
                handler = handler_factory(sock, client_id, corr_id, request)
                handler.handle_message()
            except zmq.ZMQError as err:
                if err.errno != zmq.EAGAIN:
                    LOGGER.error("Worker Error %d: %s", err.errno,
                                 zmq.strerror(err.errno))
            except Exception as exc:
                LOGGER.error("Worker Error %s\n%s", exc,
                             traceback.format_exc())
                if not handler.header_written:
                    handler.status_code = 500
                    handler.send(bytes(
                        "Worker internal error".encode('ascii')))
            finally:
                supervisor.notify_done()

            # Handle broadcast restart
            try:
                if broadcastaddr and sub.recv(flags=zmq.NOBLOCK) == b'RESTART':
                    # There is no really way to restart
                    # so exit and let the framework restart a new worker
                    LOGGER.info("Exiting on RESTART notification")
                    break
            except zmq.error.Again:
                pass
    except (KeyboardInterrupt, SystemExit):
        pass

    if broadcastaddr:
        sub.close()
    # Terminate context
    sock.close()
Example #33
0
 def test_strerror(self):
     """test that strerror gets the right type."""
     for i in range(10):
         e = strerror(i)
         self.assertTrue(isinstance(e, str))
Example #34
0
    def subscribe_run(self):
        """subscribe run: start a thread to listen for messages,
        """
        self.dhelp['run'] = __doc__
        assert 'sOnlineRouting' in self.ocmd2.oCurrentSubTarget.keys(), \
               "ERROR: " +'sOnlineRouting' + " not in " +repr(self.ocmd2.oCurrentSubTarget.keys())
        assert self.ocmd2.oCurrentSubTarget['sOnlineRouting']
        sOnlineRouting = self.ocmd2.oCurrentSubTarget['sOnlineRouting']
        sChartId = self.ocmd2.sDefaultChart
        if sOnlineRouting == 'RabbitMQ':
            try:
                # PikaListenerThread needs PikaMixin
                import OpenTrader.PikaListenerThread as ListenerThread
            except ImportError as e:
                self.vError(
                    "Cant import PikaListenerThread: add the MQL4/Python directory to PYTHONPATH? "
                    + str(e))
                self.vError("sys.path is: " + repr(sys.path))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
            sQueueName = self.ocmd2.oCurrentSubTarget['sQueueName']
        elif sOnlineRouting == 'ZeroMQ':
            try:
                import OpenTrader.ZmqListenerThread as ListenerThread
            except ImportError as e:
                self.vError(
                    "Cant import ZmqListenerThread: add the MQL4/Python directory to PYTHONPATH? "
                    + str(e))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        else:
            raise RuntimeError(
                "sOnlineRouting value in %s section of Cmd2.ini not supported"
                % (sOnlineRouting, ))

        if self.ocmd2.oListenerThread is not None:
            self.vWarn("ListenerThread already listening to: " +
                       repr(self.ocmd2.oListenerThread.lTopics))
            return
        #? Off by one
        lArgs = self.lArgs
        if len(lArgs) > 1:
            self.ocmd2.lTopics = lArgs[1:]
        else:
            self.ocmd2.lTopics = ['']
        if sOnlineRouting == 'RabbitMQ':
            from pika import exceptions
            try:
                dConfig = self.ocmd2.oConfig['RabbitMQ']
                assert 'sQueueName' in dConfig, \
                       "ERROR: sQueueName not in dConfig"
                self.ocmd2.oListenerThread = ListenerThread.PikaListenerThread(
                    sChartId, self.ocmd2.lTopics, **dConfig)
                self.ocmd2.oListenerThread.start()
            except exceptions.AMQPConnectionError as e:
                self.vError("Is the RabbitMQ server running?\n" + str(e))
                raise
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        elif sOnlineRouting == 'ZeroMQ':
            import zmq
            try:
                dConfig = self.ocmd2.oConfig['ZeroMQ']
                self.ocmd2.oListenerThread = ListenerThread.ZmqListenerThread(
                    sChartId, self.ocmd2.lTopics, **dConfig)
                self.ocmd2.oListenerThread.start()
            except zmq.ZMQError as e:
                # zmq4: iError = zmq.zmq_errno()
                iError = e.errno
                self.vWarn("run ZMQError in oListenerThread.start : %d %s" % (
                    iError,
                    zmq.strerror(iError),
                ))
            except Exception as e:
                self.vError(traceback.format_exc(10))
                raise
        return
Example #35
0
def run_worker(address: str,
               handler_factory: Type[RequestHandler],
               identity: Optional[bytes] = None,
               broadcastaddr: Optional[str] = None,
               postprocess=Callable[[], None]) -> None:
    """ Enter the message loop
    """
    ctx = zmq.Context.instance()

    sock = dealer_socket(ctx, address, identity)
    if broadcastaddr:
        sub = broadcast_socket(ctx, broadcastaddr)

    # Initialize supervisor client
    supervisor = SupervisorClient()

    def get():
        client_id, corr_id, request = sock.recv_multipart()
        LOGGER.debug("RCV %s: %s", client_id, corr_id)
        return client_id, corr_id, pickle.loads(request)

    try:
        LOGGER.info("Starting ZMQ worker loop")
        while True:
            sock.send(WORKER_READY)
            idle = False
            handler = None
            try:
                client_id, corr_id, request = get()
                supervisor.notify_busy()
                handler = handler_factory(sock, client_id, corr_id, request)
                handler.handle_message()
            except zmq.error.Again:
                idle = True
                pass
            except zmq.ZMQError as err:
                LOGGER.error("Worker Error %d: %s", err.errno,
                             zmq.strerror(err.errno))
            except Exception as exc:
                LOGGER.error("Worker Error %s\n%s", exc)
                # Print trace outside LOGGER because
                # logging output in sub-processe
                # is not captured by pytest
                traceback.print_exc()
                if handler and not handler.header_written:
                    handler.status_code = 500
                    handler.send("Worker internal error".encode())
                    # Got error 500, do not presume worker state
                    break
            finally:
                supervisor.notify_done()

            # Handle broadcast notifications
            try:
                if broadcastaddr:
                    msg = sub.recv(flags=zmq.NOBLOCK)
                    if msg == b'RESTART':
                        # There is no really way to restart
                        # so exit and let the framework restart a new worker
                        LOGGER.info("Exiting on RESTART notification")
                        break
                    elif msg == b'REPORT':
                        # Reporting asked
                        supervisor.send_report(handler_factory.get_report())
            except zmq.error.Again:
                pass

            try:
                # Run callbacks
                if postprocess:
                    postprocess(idle)
            except Exception:
                LOGGER.critical("Unhandled exception:\n%s",
                                traceback.format_exc())

    except (KeyboardInterrupt, SystemExit):
        pass

    if broadcastaddr:
        sub.close()
    sock.close()
    LOGGER.info("Terminating Worker")