Beispiel #1
0
 def test_unicode_sockopts(self):
     """test setting/getting sockopts with unicode strings"""
     topic = "tést"
     if str is not unicode:
         topic = topic.decode('utf8')
     p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
     self.assertEquals(s.send_unicode, s.send_unicode)
     self.assertEquals(p.recv_unicode, p.recv_unicode)
     self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
     if zmq.zmq_version() < '4.0.0':
         self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
         s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
     self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
     s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
     self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
     self.assertRaises(TypeError, s.getsockopt_unicode, zmq.SUBSCRIBE)
     if zmq.zmq_version() >= '4.0.0':
         # skip the rest on 4.0, because IDENTITY was removed
         return
     st = s.getsockopt(zmq.IDENTITY)
     self.assertEquals(st.decode('utf16'), s.getsockopt_unicode(zmq.IDENTITY, 'utf16'))
     time.sleep(0.1) # wait for connection/subscription
     p.send_unicode(topic,zmq.SNDMORE)
     p.send_unicode(topic*2, encoding='latin-1')
     self.assertEquals(topic, s.recv_unicode())
     self.assertEquals(topic*2, s.recv_unicode(encoding='latin-1'))
Beispiel #2
0
 def test_single_socket_forwarder_bind(self):
     if zmq.zmq_version() in ('4.1.1', '4.0.6'):
         raise SkipTest("libzmq-%s broke single-socket devices" % zmq.zmq_version())
     dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1)
     # select random port:
     binder = self.context.socket(zmq.REQ)
     port = binder.bind_to_random_port('tcp://127.0.0.1')
     binder.close()
     time.sleep(0.1)
     req = self.context.socket(zmq.REQ)
     req.connect('tcp://127.0.0.1:%i'%port)
     dev.bind_in('tcp://127.0.0.1:%i'%port)
     dev.start()
     time.sleep(.25)
     msg = b'hello'
     req.send(msg)
     self.assertEqual(msg, self.recv(req))
     del dev
     req.close()
     dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1)
     # select random port:
     binder = self.context.socket(zmq.REQ)
     port = binder.bind_to_random_port('tcp://127.0.0.1')
     binder.close()
     time.sleep(0.1)
     req = self.context.socket(zmq.REQ)
     req.connect('tcp://127.0.0.1:%i'%port)
     dev.bind_in('tcp://127.0.0.1:%i'%port)
     dev.start()
     time.sleep(.25)
     msg = b'hello again'
     req.send(msg)
     self.assertEqual(msg, self.recv(req))
     del dev
     req.close()
Beispiel #3
0
 def test_router_router(self):
     """test router-router MQ devices"""
     if zmq.zmq_version() >= '4.0.0':
         raise SkipTest("Only for libzmq < 4")
     dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, asbytes('in'), asbytes('out'))
     dev.setsockopt_in(zmq.LINGER, 0)
     dev.setsockopt_out(zmq.LINGER, 0)
     dev.setsockopt_mon(zmq.LINGER, 0)
     
     binder = self.context.socket(zmq.DEALER)
     porta = binder.bind_to_random_port('tcp://127.0.0.1')
     portb = binder.bind_to_random_port('tcp://127.0.0.1')
     binder.close()
     time.sleep(0.1)
     a = self.context.socket(zmq.DEALER)
     a.identity = asbytes('a')
     b = self.context.socket(zmq.DEALER)
     b.identity = asbytes('b')
     
     a.connect('tcp://127.0.0.1:%i'%porta)
     dev.bind_in('tcp://127.0.0.1:%i'%porta)
     b.connect('tcp://127.0.0.1:%i'%portb)
     dev.bind_out('tcp://127.0.0.1:%i'%portb)
     dev.start()
     time.sleep(0.2)
     msg = [ asbytes(m) for m in ('hello', 'there')]
     a.send_multipart([asbytes('b')]+msg)
     bmsg = self.recv_multipart(b)
     self.assertEquals(bmsg, [asbytes('a')]+msg)
     b.send_multipart(bmsg)
     amsg = self.recv_multipart(a)
     self.assertEquals(amsg, [asbytes('b')]+msg)
Beispiel #4
0
 def __init__(self):
   Tool.__init__(self)
   
   # * Initialize ZMQ context and open subscriber, publisher sockets
   self.logger.debug("ZMQ version: {}, PyZMQ version: {}".format(zmq.zmq_version(), zmq.pyzmq_version()))
   # ** Context
   self.zmqContext = zmq.Context()
   # ** Subscriber
   self.subSocket = self.zmqContext.socket(zmq.SUB)
   self.subSocket.connect(self.sub_address)
   time.sleep(0.005)  # mandatory sleep for ZMQ backend
   self.logger.debug("[sub] Connected to {}".format(self.sub_address))
   # ** Subscriber topics for input messages
   self.subSocket.setsockopt(zmq.SUBSCRIBE, self.sub_topic)
   self.subSocket.setsockopt(zmq.LINGER, self.sub_socket_linger)
   self.logger.debug("[sub]Subscribed to topic \"{}\"".format(self.sub_topic))
   time.sleep(0.005)  # mandatory sleep for ZMQ backend
   # ** Publisher
   self.pubSocket = self.zmqContext.socket(zmq.PUB)
   self.pubSocket.bind(self.pub_address)
   time.sleep(0.005)  # mandatory sleep for ZMQ backend
   self.logger.debug("[pub] Bound to {}".format(self.pub_address))
   
   # * Initialize other members
   self.valid = False
   self.buttons = [0, 0]  # primary, secondary
   self.transform = hm.translation(hm.identity(), self.position_offset)
   #self.position = self.position_offset
   self.loop = True  # TODO ensure this is properly shared across threads
   
   # * Start sensing loop
   self.senseThread = Thread(target=self.senseLoop)
   self.senseThread.daemon = True  # to prevent indefinite wait on recv()
   self.senseThread.start()
   time.sleep(0.005)  # sleep to allow child thread to run
Beispiel #5
0
 def test_labels(self):
     """test device support for SNDLABEL"""
     if zmq.zmq_version() < '3.0.0':
         raise SkipTest("Only for libzmq 3")
     dev = devices.ThreadDevice(zmq.QUEUE, zmq.XREP, -1)
     # select random port:
     binder = self.context.socket(zmq.XREQ)
     port = binder.bind_to_random_port('tcp://127.0.0.1')
     binder.close()
     time.sleep(0.1)
     req = self.context.socket(zmq.REQ)
     req.connect('tcp://127.0.0.1:%i'%port)
     dev.bind_in('tcp://127.0.0.1:%i'%port)
     dev.start()
     time.sleep(.25)
     msg = asbytes('hello')
     req.send(msg, zmq.SNDLABEL)
     req.send(msg, zmq.SNDMORE)
     req.send(msg)
     
     self.assertEquals(msg, self.recv(req))
     self.assertTrue(req.rcvlabel)
     self.assertEquals(msg, self.recv(req))
     self.assertTrue(req.rcvmore)
     self.assertEquals(msg, self.recv(req))
     del dev
     req.close()
Beispiel #6
0
    def test_router_router(self):
        """test router-router MQ devices"""
        if zmq.zmq_version() >= "4.0.0":
            raise SkipTest("Only for libzmq < 4")
        dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, "in", "out")
        dev.setsockopt_in(zmq.LINGER, 0)
        dev.setsockopt_out(zmq.LINGER, 0)
        dev.setsockopt_mon(zmq.LINGER, 0)

        binder = self.context.socket(zmq.DEALER)
        porta = binder.bind_to_random_port("tcp://127.0.0.1")
        portb = binder.bind_to_random_port("tcp://127.0.0.1")
        binder.close()
        time.sleep(0.1)
        a = self.context.socket(zmq.DEALER)
        a.identity = asbytes("a")
        b = self.context.socket(zmq.DEALER)
        b.identity = asbytes("b")

        a.connect("tcp://127.0.0.1:%i" % porta)
        dev.bind_in("tcp://127.0.0.1:%i" % porta)
        b.connect("tcp://127.0.0.1:%i" % portb)
        dev.bind_out("tcp://127.0.0.1:%i" % portb)
        dev.start()
        time.sleep(0.2)
        msg = [asbytes(m) for m in ("hello", "there")]
        a.send_multipart(["b"] + msg)
        bmsg = self.recv_multipart(b)
        self.assertEquals(bmsg, ["a"] + msg)
        b.send_multipart(bmsg)
        amsg = self.recv_multipart(a)
        self.assertEquals(amsg, ["b"] + msg)
Beispiel #7
0
 def test_prefix(self):
     if zmq.zmq_version() < '3.0.0':
         raise SkipTest("Only applies to libzmq >= 3.0")
     xrep, xreq = self.create_bound_pair(zmq.XREP, zmq.XREQ)
     msg = [ asbytes(p) for p in 'hi there'.split() ]
     xreq.send_multipart(msg)
     recvd = xrep.recv_multipart()
     self.assertTrue(isinstance(recvd, tuple))
     self.assertEquals(len(recvd), 2)
     prefix, real = recvd
     self.assertTrue(isinstance(prefix, list))
     self.assertEquals(len(prefix), 1)
     self.assertEquals(real, msg)
     xrep.send_multipart(real, prefix=prefix)
     echo = xreq.recv_multipart()
     self.assertTrue(isinstance(echo, list))
     self.assertEquals(echo, real)
     extra = [asbytes('pre')]
     xrep.send_multipart(msg, prefix=prefix+extra)
     recvd = xreq.recv_multipart()
     self.assertTrue(isinstance(recvd, tuple))
     self.assertEquals(len(recvd), 2)
     prefix, real = recvd
     self.assertTrue(isinstance(prefix, list))
     self.assertEquals(len(prefix), 1)
     self.assertEquals(prefix, extra)
     self.assertEquals(real, msg)
Beispiel #8
0
def check_for_zmq(minimum_version, module='IPython.zmq'):
    min_vlist = [int(n) for n in minimum_version.split('.')]

    try:
        import zmq
    except ImportError:
        raise ImportError("%s requires pyzmq >= %s"%(module, minimum_version))

    pyzmq_version = zmq.__version__
    vlist = [int(n) for n in re.findall(r'\d+', pyzmq_version)]

    if 'dev' not in pyzmq_version and vlist < min_vlist:
        raise ImportError("%s requires pyzmq >= %s, but you have %s"%(
                        module, minimum_version, pyzmq_version))

    # fix missing DEALER/ROUTER aliases in pyzmq < 2.1.9
    if not hasattr(zmq, 'DEALER'):
        zmq.DEALER = zmq.XREQ
    if not hasattr(zmq, 'ROUTER'):
        zmq.ROUTER = zmq.XREP

    if zmq.zmq_version() >= '4.0.0':
        warnings.warn("""libzmq 4 detected.
        It is unlikely that IPython's zmq code will work properly.
        Please install libzmq stable, which is 2.1.x or 2.2.x""",
        RuntimeWarning)
Beispiel #9
0
    def enable_monitor(self, events=None):

        # The standard approach of binding and then connecting does not
        # work in this specific case. The event loop does not properly
        # detect messages on the inproc transport which means that event
        # messages get missed.
        # pyzmq's 'get_monitor_socket' method can't be used because this
        # performs the actions in the wrong order for use with an event
        # loop.
        # For more information on this issue see:
        # http://lists.zeromq.org/pipermail/zeromq-dev/2015-July/029181.html

        if (zmq.zmq_version_info() < (4,) or
                zmq.pyzmq_version_info() < (14, 4,)):
            raise NotImplementedError(
                "Socket monitor requires libzmq >= 4 and pyzmq >= 14.4, "
                "have libzmq:{}, pyzmq:{}".format(
                    zmq.zmq_version(), zmq.pyzmq_version()))

        if self._monitor is None:
            addr = "inproc://monitor.s-{}".format(self._zmq_sock.FD)
            events = events or zmq.EVENT_ALL
            _, self._monitor = yield from create_zmq_connection(
                lambda: _ZmqEventProtocol(self._loop, self._protocol),
                zmq.PAIR, connect=addr, loop=self._loop)
            # bind must come after connect
            self._zmq_sock.monitor(addr, events)
            yield from self._monitor.wait_ready
    def _printheader(self, log):
        """Prints header to log file; inspired by that in GPAW."""
        log(logo)
        log('Amp: Atomistic Machine-learning Package')
        log('Developed by Andrew Peterson, Alireza Khorshidi, and others,')
        log('Brown University.')
        log(' PI Website: http://brown.edu/go/catalyst')
        log(' Official repository: http://bitbucket.org/andrewpeterson/amp')
        log(' Official documentation: http://amp.readthedocs.org/')
        log(' Citation:')
        log('  Khorshidi & Peterson, Computer Physics Communications')
        log('  doi:10.1016/j.cpc.2016.05.010 (2016)')
        log('=' * 70)
        log('User: %s' % getuser())
        log('Hostname: %s' % gethostname())
        log('Date: %s' % now(with_utc=True))
        uname = platform.uname()
        log('Architecture: %s' % uname[4])
        log('PID: %s' % os.getpid())
        log('Amp version: %s' % 'NOT NUMBERED YET.')  # FIXME/ap. Look at GPAW
        ampdirectory = os.path.dirname(os.path.abspath(__file__))
        log('Amp directory: %s' % ampdirectory)
        commithash, commitdate = get_git_commit(ampdirectory)
        log(' Last commit: %s' % commithash)
        log(' Last commit date: %s' % commitdate)
        log('Python: v{0}.{1}.{2}: %s'.format(*sys.version_info[:3]) %
            sys.executable)
        log('ASE v%s: %s' % (aseversion, os.path.dirname(ase.__file__)))
        log('NumPy v%s: %s' %
            (np.version.version, os.path.dirname(np.__file__)))
        # SciPy is not a strict dependency.
        try:
            import scipy
            log('SciPy v%s: %s' %
                (scipy.version.version, os.path.dirname(scipy.__file__)))
        except ImportError:
            log('SciPy: not available')
        # ZMQ an pxssh are only necessary for parallel calculations.
        try:
            import zmq
            log('ZMQ/PyZMQ v%s/v%s: %s' %
                (zmq.zmq_version(), zmq.pyzmq_version(),
                 os.path.dirname(zmq.__file__)))
        except ImportError:
            log('ZMQ: not available')
        try:
            import pxssh
            log('pxssh: %s' % os.path.dirname(pxssh.__file__))
        except ImportError:
            log('pxssh: Not available from pxssh.')
            try:
                from pexpect import pxssh
            except ImportError:
                log('pxssh: Not available from pexpect.')
            else:
                import pexpect
                log('pxssh (via pexpect v%s): %s' %
                    (pexpect.__version__, pxssh.__file__))

        log('=' * 70)
Beispiel #11
0
    def run(self):
        """Run the test suite, with nose, or unittest if nose is unavailable"""
        # crude check for inplace build:
        try:
            import zmq
        except ImportError:
            print_exc()
            fatal(
                "\n       ".join(
                    [
                        "Could not import zmq!",
                        "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
                        "If you did build pyzmq in-place, then this is a real error.",
                    ]
                )
            )
            sys.exit(1)

        info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))

        if nose is None:
            warn("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
            return self.run_unittest()
        else:
            return self.run_nose()
Beispiel #12
0
def print_info():
    log.info("Python version: %s, %s",
             '.'.join((str(e) for e in sys.version_info)),
             sys.executable)

    log.info("zeromq version: %s", zmq.zmq_version())
    log.info("pyzmq version:  %s", zmq.pyzmq_version())
    log.info("track version:  %s", str(track_base.version_info))
Beispiel #13
0
 def __init__(self, min_version, msg='Feature'):
     global _zmq_version
     if _zmq_version is None:
         from zmq import zmq_version
         _zmq_version = zmq_version()
     self.msg = msg
     self.min_version = min_version
     self.version = _zmq_version
Beispiel #14
0
def zmqversion():
    '''
    Return the zeromq version
    '''
    # Provides:
    #   zmqversion
    import zmq
    return {'zmqversion': zmq.zmq_version()}
Beispiel #15
0
def log_versions(logger):
    logger.info(
        'Versions:\n'
        '  PyZMQ: %s\n'
        '  libzmq: %s\n'
        '  Tornado: %s\n'
        '  SockJS-Tornado: Not Available\n'  # TODO find the verison
        '  SockJS-ZMQProxy: %s\n',
        zmq.pyzmq_version(),
        zmq.zmq_version(),
        tornado_version,
        version
    )
Beispiel #16
0
    def test_bad_send_recv(self):
        s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP)

        if zmq.zmq_version() != "2.1.8":
            # this doesn't work on 2.1.8
            for copy in (True, False):
                self.assertRaisesErrno(zmq.EFSM, s1.recv, copy=copy)
                self.assertRaisesErrno(zmq.EFSM, s2.send, b"asdf", copy=copy)

        # I have to have this or we die on an Abort trap.
        msg1 = b"asdf"
        msg2 = self.ping_pong(s1, s2, msg1)
        self.assertEqual(msg1, msg2)
Beispiel #17
0
    def test_router_dealer(self):
        if zmq.zmq_version() >= '3.0.0':
            raise SkipTest("Known bug in libzmq 3.0.0, see https://zeromq.jira.com/browse/LIBZMQ-232")
        router, dealer = self.create_bound_pair(zmq.ROUTER, zmq.DEALER)

        msg1 = asbytes('message1')
        dealer.send(msg1)
        ident = router.recv()
        more = router.rcvmore()
        self.assertEquals(more, True)
        msg2 = router.recv()
        self.assertEquals(msg1, msg2)
        more = router.rcvmore()
        self.assertEquals(more, False)
Beispiel #18
0
    def test_router_dealer(self):
        if zmq.zmq_version() >= '4.0.0':
            raise SkipTest("ROUTER/DEALER change in 4.0")
        router, dealer = self.create_bound_pair(zmq.ROUTER, zmq.DEALER)

        msg1 = asbytes('message1')
        dealer.send(msg1)
        ident = self.recv(router)
        more = router.rcvmore
        self.assertEquals(more, True)
        msg2 = self.recv(router)
        self.assertEquals(msg1, msg2)
        more = router.rcvmore
        self.assertEquals(more, False)
Beispiel #19
0
 def test_int_sockopts(self):
     "test non-uint64 sockopts"
     v = zmq.zmq_version()
     if not v >= '2.1':
         raise SkipTest("only on libzmq >= 2.1")
     elif v < '3.0':
         hwm = zmq.HWM
         default_hwm = 0
     else:
         hwm = zmq.SNDHWM
         default_hwm = 1000
     p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
     p.setsockopt(zmq.LINGER, 0)
     self.assertEquals(p.getsockopt(zmq.LINGER), 0)
     p.setsockopt(zmq.LINGER, -1)
     self.assertEquals(p.getsockopt(zmq.LINGER), -1)
     self.assertEquals(p.getsockopt(hwm), default_hwm)
     p.setsockopt(hwm, 11)
     self.assertEquals(p.getsockopt(hwm), 11)
     # p.setsockopt(zmq.EVENTS, zmq.POLLIN)
     self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
     self.assertRaisesErrno(zmq.EINVAL, p.setsockopt,zmq.EVENTS, 2**7-1)
     self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
     self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
     self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
     self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
     
     # check for overflow / wrong type:
     errors = []
     backref = {}
     constants = zmq.core.constants
     for name in constants.__all__:
         value = getattr(constants, name)
         if isinstance(value, int):
             backref[value] = name
     for opt in zmq.core.constants.int_sockopts+zmq.core.constants.int64_sockopts:
         sopt = backref[opt]
         try:
             n = p.getsockopt(opt)
         except zmq.ZMQError:
             e = sys.exc_info()[1]
             errors.append("getsockopt(zmq.%s) raised '%s'."%(sopt, e))
         else:
             if n > 2**31:
                 errors.append("getsockopt(zmq.%s) returned a ridiculous value."
                                 " It is probably the wrong type."%sopt)
     if errors:
         self.fail('\n'.join(errors))
Beispiel #20
0
def test_closing_full():
    ctx				= zmq.Context()
    ver				= zmq.zmq_version()[:3]

    global port
    out 			= ctx.socket( zmq.XREQ )
    out.bind( "tcp://*:%d" % port )

    inc				= ctx.socket( zmq.XREP )
    inc.connect( "tcp://localhost:%d" % port )
    port	       	       += 1

    try:
        # Test that ..._multipart and zmq.RCVMORE behave as we think
        out.send_multipart( ['', 'TEST1'] )
        msg			= inc.recv_multipart()
        if ver in ("2.1", "3.1"):
            assert len( msg ) == 3    
        elif ver in ("3.0"):
            assert isinstance( msg, tuple )
            assert len( msg[0] ) == 1
            assert len( msg[1] ) == 2
        else:
            assert False and "Unknown zmq version"
        
        out.send_multipart( ['', 'TEST2'] )
        msg 			= []
        poller			= zmq.Poller()
        poller.register(inc, zmq.POLLIN)
        while poller.poll( timeout=100 ):
            msg.append( inc.recv() )
            if not inc.getsockopt( zmq.RCVMORE ):
                break
        assert len( msg ) == 3
        
        ## Add another stage; see if we can intercept and re-route between
        ## inc/nxt
        #nxt			= ctx.socket( zmq.XREQ )
        #nxt.connect( "tcp://localhost:%d" % port )
        #
        #lst			= ctx.socket( zme.XREP )
        #nxt.bind( "tcp://*:%d" % port )
        #port	       	       += 1
        
    finally:
        out.close()
        inc.close()
        ctx.term()
Beispiel #21
0
 def run(self):
     """Run the test suite with py.test"""
     # crude check for inplace build:
     try:
         import zmq
     except ImportError:
         print_exc()
         fatal('\n       '.join(["Could not import zmq!",
         "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
         "If you did build pyzmq in-place, then this is a real error."]))
         sys.exit(1)
     
     info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
     p = Popen([sys.executable, '-m', 'pytest', '-v', os.path.join('zmq', 'tests')])
     p.wait()
     sys.exit(p.returncode)
Beispiel #22
0
def check_for_zmq(minimum_version, module='IPython.zmq'):
    try:
        import zmq
    except ImportError:
        raise ImportError("%s requires pyzmq >= %s"%(module, minimum_version))

    pyzmq_version = zmq.__version__
    
    if 'dev' not in pyzmq_version and V(pyzmq_version) < V(minimum_version):
        raise ImportError("%s requires pyzmq >= %s, but you have %s"%(
                        module, minimum_version, pyzmq_version))

    if V(zmq.zmq_version()) >= V('4.0.0'):
        warnings.warn("""libzmq 4 detected.
        It is unlikely that IPython's zmq code will work properly.
        Please install libzmq stable, which is 2.1.x or 2.2.x""",
        RuntimeWarning)
Beispiel #23
0
 def level_1(info):
     # print header (some general info)
     info += ["name: %s" % self.job_name]
     info += ["version: %s" % getattr(self, "__version__", "")]
     info += ["service: %s" % self.service_name]
     info += ["pid: %i" % os.getpid()]
     info += ["started: %s" % strftime(getattr(self, "started", ""))]
     info += ["status: %s" % getattr(self, "status", "")]
     info += ["time-consumed: %s" % " ".join(map(str, os.times()[:4]))]
     info += ["info-period: %s" % self.stats_period]
     info += ["info-written: %s" % strftime(time.time())]
     info += ["platform: %s" % platform.platform()]
     info += ["python: %s" % platform.python_version()]
     info += ["skytools: %s" % skytools.__version__]
     info += ["libzmq: %s" % zmq.zmq_version()]
     info += ["pyzmq: %s" % zmq.pyzmq_version()]
     info += [""]
Beispiel #24
0
 def test_2_1_sockopts(self):
     "test non-uint64 sockopts introduced in zeromq 2.1.0"
     v = list(map(int, zmq.zmq_version().split('.', 2)[:2]))
     if not (v[0] >= 2 and v[1] >= 1):
         raise SkipTest
     p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
     p.setsockopt(zmq.LINGER, 0)
     self.assertEquals(p.getsockopt(zmq.LINGER), 0)
     p.setsockopt(zmq.LINGER, -1)
     self.assertEquals(p.getsockopt(zmq.LINGER), -1)
     # p.setsockopt(zmq.EVENTS, zmq.POLLIN)
     self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
     self.assertRaisesErrno(zmq.EINVAL, p.setsockopt,zmq.EVENTS, 2**7-1)
     self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
     self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
     self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
     self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
Beispiel #25
0
def set_tcp_keepalive(sock, opts=None):
    # Warn if ZMQ < 3.2
    try:
        zmq_version_info = zmq.zmq_version_info()
    except AttributeError:
        # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
        # using zmq.zmq_version() and build a version info tuple.
        zmq_version_info = tuple(
            [int(x) for x in zmq.zmq_version().split('.')]
        )
    if zmq_version_info < (3, 2):
        log.warning(
            'You have a version of ZMQ less than ZMQ 3.2! There are '
            'known connection keep-alive issues with ZMQ < 3.2 which '
            'may result in loss of contact with minions. Please '
            'upgrade your ZMQ!'
        )
    ####
    # try:
    #     ip=socket.gethostbyname(socket.gethostname())
    #     if ip == "127.0.0.1":
    #         raise Exception("127.0.0.1")
    # except Exception,e:
    #     (status,ip) = commands.getstatusoutput("/sbin/ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d 'addr:'")
    #     ip = ip
    if opts['ha'] == True:
        while True:
            time.sleep(5)
            try:
                sys.path.append("/usr/lib/python2.7/site-packages/salt")
                from zkclient import ZKClient, zookeeper, watchmethod
                zk = ZKClient('172.16.163.14:2181',1000)
                if zk.exists('/broker'):
                    value = zk.get('/broker')
                    logging.error(str(value[0]))
                    if value[0] == opts['id']:
                        break
                else:
                    zk.create('/broker')
                    zk.set('/broker',data='%s'%opts['id'])
                    logging.error("create the zk path is successful !")
                    continue
            except Exception,e:
                logging.error(str(e))
                continue
Beispiel #26
0
    def setup(self):
        sock = self.socket
        sock.identity = identity = str(uuid.uuid4())
        _log.debug("ROUTER SOCK identity: {}".format(sock.identity))
        if self._monitor:
            Monitor(sock.get_monitor_socket()).start()
        sock.bind('inproc://vip')
        _log.debug('In-process VIP router bound to inproc://vip')
        sock.zap_domain = 'vip'
        addr = self.local_address
        if not addr.identity:
            addr.identity = identity
        if not addr.domain:
            addr.domain = 'vip'

        addr.server = 'CURVE'
        addr.secretkey = self._secretkey

        addr.bind(sock)
        _log.debug('Local VIP router bound to %s' % addr)
        for address in self.addresses:
            if not address.identity:
                address.identity = identity
            if (address.secretkey is None
                    and address.server not in ['NULL', 'PLAIN']
                    and self._secretkey):
                address.server = 'CURVE'
                address.secretkey = self._secretkey
            if not address.domain:
                address.domain = 'vip'
            address.bind(sock)
            _log.debug('Additional VIP router bound to %s' % address)
        self._ext_routing = None

        self._ext_routing = RoutingService(self.socket, self.context,
                                           self._socket_class, self._poller,
                                           self._addr, self._instance_name)

        self._pubsub = PubSubService(self.socket, self._protected_topics,
                                     self._ext_routing)
        self._ext_rpc = ExternalRPCService(self.socket, self._ext_routing)
        self._poller.register(sock, zmq.POLLIN)
        _log.debug("ZMQ version: {}".format(zmq.zmq_version()))
Beispiel #27
0
    def get_monitor_socket(self, events=None, addr=None):
        """Return a connected PAIR socket ready to receive the event notifications.
        
        .. versionadded:: libzmq-4.0
        .. versionadded:: 14.0
        
        Parameters
        ----------
        events : int [default: ZMQ_EVENTS_ALL]
            The bitmask defining which events are wanted.
        addr :  string [default: None]
            The optional endpoint for the monitoring sockets.

        Returns
        -------
        socket :  (PAIR)
            The socket is already connected and ready to receive messages.
        """
        # safe-guard, method only available on libzmq >= 4
        if zmq.zmq_version_info() < (4, ):
            raise NotImplementedError(
                "get_monitor_socket requires libzmq >= 4, have %s" %
                zmq.zmq_version())

        # if already monitoring, return existing socket
        if self._monitor_socket:
            if self._monitor_socket.closed:
                self._monitor_socket = None
            else:
                return self._monitor_socket

        if addr is None:
            # create endpoint name from internal fd
            addr = "inproc://monitor.s-%d" % self.FD
        if events is None:
            # use all events
            events = zmq.EVENT_ALL
        # attach monitoring socket
        self.monitor(addr, events)
        # create new PAIR socket and connect it
        self._monitor_socket = self.context.socket(zmq.PAIR)
        self._monitor_socket.connect(addr)
        return self._monitor_socket
Beispiel #28
0
 def __init__(self, opts):
     '''
     Create a salt broker instance
     '''
     self.opts = opts
     # Warn if ZMQ < 3.2
     try:
         zmq_version_info = zmq.zmq_version_info()
     except AttributeError:
         # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
         # using zmq.zmq_version() and build a version info tuple.
         zmq_version_info = tuple(
             [int(x) for x in zmq.zmq_version().split('.')])
     if zmq_version_info < (3, 2):
         log.warning(
             'You have a version of ZMQ less than ZMQ 3.2! There are '
             'known connection keep-alive issues with ZMQ < 3.2 which '
             'may result in loss of contact with minions. Please '
             'upgrade your ZMQ!')
Beispiel #29
0
    def __init__(self, args):
        super().__init__()
        self.logger = set_logger(colored('VENTILATOR', 'magenta'),
                                 args.verbose)

        self.model_dir = args.model_dir
        self.spm_model_file = args.spm_model_file
        self.max_seq_len = args.max_seq_len
        self.num_worker = args.num_worker
        self.max_batch_size = args.max_batch_size
        self.num_concurrent_socket = max(
            8, args.num_worker * 2)  # optimize concurrency for multi-clients
        self.port = args.port
        self.args = args
        self.status_args = {
            k: (v if k != 'pooling_strategy' else v.value)
            for k, v in sorted(vars(args).items())
        }
        self.status_static = {
            'tensorflow_version': _tf_ver_,
            'python_version': sys.version,
            'server_version': __version__,
            'pyzmq_version': zmq.pyzmq_version(),
            'zmq_version': zmq.zmq_version(),
            'server_start_time': str(datetime.now()),
        }
        self.processes = []
        self.logger.info(
            'freeze, optimize and export graph, could take a while...')
        with Pool(processes=1) as pool:
            # optimize the graph, must be done in another process
            from .graph import optimize_graph
            self.graph_path, self.bert_config = pool.apply(
                optimize_graph, (self.args, ))
        # from .graph import optimize_graph
        # self.graph_path = optimize_graph(self.args, self.logger)
        if self.graph_path:
            self.logger.info('optimized graph is stored at: %s' %
                             self.graph_path)
        else:
            raise FileNotFoundError(
                'graph optimization fails and returns empty result')
        self.is_ready = threading.Event()
def main():
    context = zmq.Context()
    socket = context.socket(zmq.PUSH)
    socket.copy_threshold = 0
    socket.connect("tcp://localhost:50300")

    print("using zmq version", zmq.__version__)
    print("using libzmq version", zmq.zmq_version())

    message = b"World"
    print("Send: ", message)
    res = socket.send(message, copy=False, track=True)

    if res.done:
        print("res: done")
    else:
        print("res: waiting")
        res.wait()
        print("res: waiting...")
Beispiel #31
0
 def __init__(self, port=5558, port_out=5559, n_workers=1, verbose=False,
              max_batch_size=32, task='coref'):
     super().__init__()
     self.logger = set_logger(colored('VENTILATOR', 'magenta'), verbose)
     self.port = port
     self.port_out = port_out
     self.processes = []
     self.is_ready = threading.Event()
     self.n_workers = n_workers
     self.n_concurrent_sockets = max(8, n_workers * 2)
     self.max_batch_size = max_batch_size
     self.status_static = {
         'python_version': sys.version,
         'server_version': __version__,
         'pyzmq_version': zmq.pyzmq_version(),
         'zmq_version': zmq.zmq_version(),
         'server_start_time': str(datetime.now()),
     }
     self.Worker = WorkerRegistry[task]
Beispiel #32
0
 def run(self):
     """Run the test suite, with nose, or unittest if nose is unavailable"""
     # crude check for inplace build:
     try:
         import zmq
     except ImportError:
         print_exc()
         fatal('\n       '.join(["Could not import zmq!",
         "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
         "If you did build pyzmq in-place, then this is a real error."]))
         sys.exit(1)
     
     info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
     
     if nose is None:
         warn("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
         return self.run_unittest()
     else:
         return self.run_nose()
Beispiel #33
0
def get_full_version() -> Optional[Tuple[Dict, Dict]]:
    """
    Get the version of libraries used in Jina and environment variables.

    :return: Version information and environment variables
    """
    from . import __version__, __proto_version__, __jina_env__
    from google.protobuf.internal import api_implementation
    import os, zmq, numpy, google.protobuf, grpc, yaml
    from grpc import _grpcio_metadata
    from pkg_resources import resource_filename
    import platform
    from .logging import default_logger

    try:

        info = {
            'jina': __version__,
            'jina-proto': __proto_version__,
            'jina-vcs-tag': os.environ.get('JINA_VCS_VERSION', '(unset)'),
            'libzmq': zmq.zmq_version(),
            'pyzmq': numpy.__version__,
            'protobuf': google.protobuf.__version__,
            'proto-backend': api_implementation._default_implementation_type,
            'grpcio': getattr(grpc, '__version__',
                              _grpcio_metadata.__version__),
            'pyyaml': yaml.__version__,
            'python': platform.python_version(),
            'platform': platform.system(),
            'platform-release': platform.release(),
            'platform-version': platform.version(),
            'architecture': platform.machine(),
            'processor': platform.processor(),
            'jina-resources': resource_filename('jina', 'resources'),
        }
        env_info = {k: os.getenv(k, '(unset)') for k in __jina_env__}
        full_version = info, env_info
    except Exception as e:
        default_logger.error(str(e))
        full_version = None

    return full_version
Beispiel #34
0
def connect(server="localhost", port=5555):
    """
    Connect to a running arkouda server.

    Parameters
    ----------
    server : str, optional
        The hostname of the server (must be visible to the current 
        machine). Defaults to `localhost`.
    port : int, optional
        The port of the server. Defaults to 5555.

    Returns
    -------
    None
        On success, prints ``connected to tcp://<hostname>:<port>``
    """
    global verbose, context, socket, pspStr, serverPid, connected

    if connected == False:
        print(zmq.zmq_version())

        # "protocol://server:port"
        pspStr = "tcp://{}:{}".format(server, port)
        print("psp = ", pspStr)

        # setup connection to arkouda server
        context = zmq.Context()
        socket = context.socket(zmq.REQ)  # request end of the zmq connection
        socket.connect(pspStr)
        connected = True

        #send the connect message
        message = "connect"
        if verbose: print("[Python] Sending request: %s" % message)
        socket.send_string(message)

        # get the response that the server has started
        message = socket.recv_string()
        if verbose: print("[Python] Received response: %s" % message)

        print("connected to {}".format(pspStr))
Beispiel #35
0
    def setup(self):
        sock = self.socket
        sock.identity = identity = str(uuid.uuid4())
        _log.debug("ROUTER SOCK identity: {}".format(sock.identity))
        if self._monitor:
            Monitor(sock.get_monitor_socket()).start()
        sock.bind('inproc://vip')
        _log.debug('In-process VIP router bound to inproc://vip')
        sock.zap_domain = 'vip'
        addr = self.local_address
        if not addr.identity:
            addr.identity = identity
        if not addr.domain:
            addr.domain = 'vip'

        addr.server = 'CURVE'
        addr.secretkey = self._secretkey

        addr.bind(sock)
        _log.debug('Local VIP router bound to %s' % addr)
        for address in self.addresses:
            if not address.identity:
                address.identity = identity
            if (address.secretkey is None and
                        address.server not in ['NULL', 'PLAIN'] and
                    self._secretkey):
                address.server = 'CURVE'
                address.secretkey = self._secretkey
            if not address.domain:
                address.domain = 'vip'
            address.bind(sock)
            _log.debug('Additional VIP router bound to %s' % address)
        self._ext_routing = None

        self._ext_routing = RoutingService(self.socket, self.context,
                                           self._socket_class, self._poller,
                                           self._addr, self._instance_name)

        self._pubsub = PubSubService(self.socket, self._protected_topics, self._ext_routing)
        self._ext_rpc = ExternalRPCService(self.socket, self._ext_routing)
        self._poller.register(sock, zmq.POLLIN)
        _log.debug("ZMQ version: {}".format(zmq.zmq_version()))
Beispiel #36
0
 def test_2_1_sockopts(self):
     "test non-uint64 sockopts introduced in zeromq 2.1.0"
     v = list(map(int, zmq.zmq_version().split('.', 2)[:2]))
     if not (v[0] >= 2 and v[1] >= 1):
         raise SkipTest
     p, s = self.create_bound_pair(zmq.PUB, zmq.SUB)
     p.setsockopt(zmq.LINGER, 0)
     self.assertEquals(p.getsockopt(zmq.LINGER), 0)
     p.setsockopt(zmq.LINGER, -1)
     self.assertEquals(p.getsockopt(zmq.LINGER), -1)
     self.assertEquals(p.getsockopt(zmq.HWM), 0)
     p.setsockopt(zmq.HWM, 11)
     self.assertEquals(p.getsockopt(zmq.HWM), 11)
     # p.setsockopt(zmq.EVENTS, zmq.POLLIN)
     self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
     self.assertRaisesErrno(zmq.EINVAL, p.setsockopt, zmq.EVENTS, 2**7 - 1)
     self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
     self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
     self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
     self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
Beispiel #37
0
    def __init__(self, args):
        self.args = args
        self.logger = set_logger(self.__class__.__name__, self.args.verbose)
        self.ctx = zmq.Context()
        self.ctx.setsockopt(zmq.LINGER, 0)
        self.logger.info('current libzmq version is %s,  pyzmq version is %s' %
                         (zmq.zmq_version(), zmq.__version__))

        self.receiver, recv_addr = build_socket(self.ctx, self.args.host_in,
                                                self.args.port_in,
                                                self.args.socket_in,
                                                self.args.identity)
        self.sender, send_addr = build_socket(self.ctx, self.args.host_out,
                                              self.args.port_out,
                                              self.args.socket_out,
                                              self.args.identity)
        self.logger.info(
            'input %s:%s\t output %s:%s' %
            (self.args.host_in, colored(self.args.port_in, 'yellow'),
             self.args.host_out, colored(self.args.port_out, 'yellow')))
Beispiel #38
0
    def monitor(self, addr, events=-1):
        """s.monitor(addr, flags)

        Start publishing socket events on inproc.
        See libzmq docs for zmq_monitor for details.
        
        Note: requires libzmq >= 3.2
        
        Parameters
        ----------
        addr : str
            The inproc url used for monitoring.
        events : int [default: zmq.EVENT_ALL]
            The zmq event bitmask for which events will be sent to the monitor.
        """
        if zmq.zmq_version_info() < (3,2):
            raise NotImplementedError("monitor requires libzmq >= 3.2, have %s" % zmq.zmq_version())
        if events < 0:
            events = zmq.EVENT_ALL
        rc = C.zmq_socket_monitor(self._zmq_socket, addr, events)
Beispiel #39
0
def log_sysinfo(app: Flask, config: Config):
    app.logger.info("ZMQ:")
    app.logger.info("  zmq version: %s", zmq.zmq_version())
    app.logger.info("  pyzmq version: %s", zmq.pyzmq_version())
    app.logger.info("  zmq includes: %s", zmq.get_includes())
    app.logger.info("  zmq library dirs: %s", zmq.get_library_dirs())
    app.logger.info("  has: %s", [c for c in ZMQ_CAPABILITIES if zmq.has(c)])
    app.logger.info("socket:")
    app.logger.info("  fqdn: %s", socket.getfqdn())
    app.logger.info("  has_ipv6: %s", socket.has_ipv6)
    app.logger.info("  hostname: %s", socket.gethostname())
    app.logger.info("  interfaces: %s", [i[1] for i in socket.if_nameindex()])
    app.logger.info("os:")
    app.logger.info("  ctermid: %s", os.ctermid())
    app.logger.info("  cwd: %s", os.getcwd())
    app.logger.info("  groups: %s", os.getgroups())
    app.logger.info("  pgid: %d", os.getpgid(0))
    app.logger.info("  pgrp: %d", os.getpgrp())
    app.logger.info("  pid: %d", os.getpid())
    app.logger.info("  ppid: %d", os.getppid())
    app.logger.info("  priority_process: %d",
                    os.getpriority(os.PRIO_PROCESS, 0))
    app.logger.info("  priority_pgrp: %d", os.getpriority(os.PRIO_PGRP, 0))
    app.logger.info("  priority_user: %d", os.getpriority(os.PRIO_USER, 0))
    app.logger.info("  resuid: ruid=%d, euid=%d, suid=%d", *os.getresuid())
    app.logger.info("  resgid: rgid=%d, egid=%d, sgid=%d", *os.getresgid())
    app.logger.info("  sid: %d", os.getsid(0))
    app.logger.info("  supports_bytes_environ: %s", os.supports_bytes_environ)
    app.logger.info("  uname: %s", os.uname())
    app.logger.info("  cpu_count: %d", os.cpu_count())
    app.logger.info("platform:")
    app.logger.info("  %s", platform.platform())
    app.logger.info("  python_build: %s", platform.python_build())
    app.logger.info("  python_compiler: %s", platform.python_compiler())
    app.logger.info("  python_branch: %s", platform.python_branch())
    app.logger.info("  python_implementation: %s",
                    platform.python_implementation())
    app.logger.info("  python_revision: %s", platform.python_revision())
    app.logger.info("  python_version: %s", platform.python_version())
    app.logger.info("getpass:"******"  user: %s", getpass.getuser())
Beispiel #40
0
    def __init__(self, args, hardprocesser=WKRHardWorker):
        super().__init__()

        self.hardprocessor_skeleton = hardprocesser
        if not issubclass(self.hardprocessor_skeleton, WKRHardWorker):
            raise AssertionError(
                'hardprocesser must inherit from class WKRHardWorker')

        self.model_dir = args.model_dir

        self.num_worker = args.num_worker
        self.device_map = args.device_map
        self.gpu_memory_fraction = args.gpu_memory_fraction
        self.all_cpu = args.cpu

        self.num_concurrent_postsocket = max(8, args.num_worker * 2)
        self.batch_size = args.batch_size

        self.total_concurrent_socket = self.num_concurrent_postsocket

        self.port = args.port
        self.args = args
        self.transfer_protocol = args.protocol

        self.status_args = {k: v for k, v in sorted(vars(args).items())}
        self.status_static = {
            'python_version': sys.version,
            'server_version': __version__,
            'pyzmq_version': zmq.pyzmq_version(),
            'zmq_version': zmq.zmq_version(),
            'server_start_time': str(datetime.now()),
        }
        self.processes = []
        self.logdir = args.log_dir
        self.logger = set_logger(colored('NAVIGATOR', 'red'),
                                 logger_dir=self.logdir,
                                 verbose=args.verbose)
        self.logger.info(
            'freeze, optimize and export graph, could take a while...')

        self.is_ready = threading.Event()
Beispiel #41
0
 def build_device(self, mon_sub=asbytes(""), in_prefix=asbytes('in'), out_prefix=asbytes('out')):
     if zmq.zmq_version() >= '3':
         raise SkipTest("MonitoredQueues don't work reliably on libzmq >= 3.0.0")
     self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB,
                                         in_prefix, out_prefix)
     alice = self.context.socket(zmq.PAIR)
     bob = self.context.socket(zmq.PAIR)
     mon = self.context.socket(zmq.SUB)
     
     aport = alice.bind_to_random_port('tcp://127.0.0.1')
     bport = bob.bind_to_random_port('tcp://127.0.0.1')
     mport = mon.bind_to_random_port('tcp://127.0.0.1')
     mon.setsockopt(zmq.SUBSCRIBE, mon_sub)
     
     self.device.connect_in("tcp://127.0.0.1:%i"%aport)
     self.device.connect_out("tcp://127.0.0.1:%i"%bport)
     self.device.connect_mon("tcp://127.0.0.1:%i"%mport)
     time.sleep(.2)
     self.device.start()
     self.sockets.extend([alice, bob, mon])
     return alice, bob, mon
Beispiel #42
0
def _get_master_uri(master_ip, master_port, source_ip=None, source_port=None):
    '''
    Return the ZeroMQ URI to connect the Minion to the Master.
    It supports different source IP / port, given the ZeroMQ syntax:

    // Connecting using a IP address and bind to an IP address
    rc = zmq_connect(socket, "tcp://192.168.1.17:5555;192.168.1.1:5555"); assert (rc == 0);

    Source: http://api.zeromq.org/4-1:zmq-tcp
    '''
    libzmq_version_tup = tuple(map(int, zmq.zmq_version().split('.')))
    pyzmq_version_tup = tuple(map(int, zmq.pyzmq_version().split('.')))
    if libzmq_version_tup >= (4, 1, 6) and pyzmq_version_tup >= (16, 0, 1):
        # The source:port syntax for ZeroMQ has been added in libzmq 4.1.6
        # which is included in the pyzmq wheels starting with 16.0.1.
        if source_ip or source_port:
            if source_ip and source_port:
                return 'tcp://{source_ip}:{source_port};{master_ip}:{master_port}'.format(
                    source_ip=source_ip,
                    source_port=source_port,
                    master_ip=master_ip,
                    master_port=master_port)
            elif source_ip and not source_port:
                return 'tcp://{source_ip}:0;{master_ip}:{master_port}'.format(
                    source_ip=source_ip,
                    master_ip=master_ip,
                    master_port=master_port)
            elif not source_ip and source_port:
                return 'tcp://0.0.0.0:{source_port};{master_ip}:{master_port}'.format(
                    source_port=source_port,
                    master_ip=master_ip,
                    master_port=master_port)
    if source_ip or source_port:
        log.warning(
            'Unable to connect to the Master using a specific source IP / port'
        )
        log.warning(
            'Consider upgrading to pyzmq >= 16.0.1 and libzmq >= 4.1.6')
    return 'tcp://{master_ip}:{master_port}'.format(master_ip=master_ip,
                                                    master_port=master_port)
Beispiel #43
0
def package_version(package_name, destination):
    """Returns the package version string

    Args:
        package_name (str): the package name to look up
        destination (list): the list to store the result (tuple) to

    """
    try:
        version = metadata_version(package_name)
    except PackageNotFoundError:
        version = "<unavailable>"

    if package_name == "pyzmq":
        from zmq import zmq_version

        version = "{} (libzmq version {})".format(
            version,
            zmq_version(),
        )

    destination.append((package_name, version))
Beispiel #44
0
    def __init__(self, args):
        super().__init__()
        self.logger = set_logger(colored('VENTILATOR', 'magenta'), args.verbose)

        # self.model_dir = args.model_dir
        # self.max_seq_len = args.max_seq_len
        self.num_worker = args.num_worker
        self.max_batch_size = args.max_batch_size
        self.num_concurrent_socket = max(8, args.num_worker * 2)  # optimize concurrency for multi-clients
        self.port = args.port
        self.args = args
        self.status_args = {k: (v if k != 'pooling_strategy' else v.value) for k, v in sorted(vars(args).items())}
        self.status_static = {
            'python_version': sys.version,
            'pyzmq_version': zmq.pyzmq_version(),
            'zmq_version': zmq.zmq_version(),
            'server_start_time': str(datetime.now()),
        }
        self.processes = []
        self.logger.info('freeze, optimize and export graph, could take a while...')

        self.is_ready = threading.Event()
Beispiel #45
0
    def __init__(self, args):
        super().__init__()
        self.logger = set_logger(colored('VENTILATOR', 'magenta'))

        self.num_worker = args.num_worker
        self.max_batch_size = args.max_batch_size
        self.num_concurrent_socket = max(
            8, args.num_worker * 2)  # optimize concurrency for multi-clients
        self.port = args.port
        self.args = args
        self.status_args = {
            k: (v if k != 'pooling_strategy' else v.value)
            for k, v in sorted(vars(args).items())
        }
        self.status_static = {
            'python_version': sys.version,
            'server_version': __version__,
            'pyzmq_version': zmq.pyzmq_version(),
            'zmq_version': zmq.zmq_version(),
            'server_start_time': str(datetime.now()),
        }
        self.processes = []
Beispiel #46
0
    def run(self):
        """Run the test suite with py.test"""
        # crude check for inplace build:
        try:
            import zmq
        except ImportError:
            print_exc()
            fatal('\n       '.join([
                "Could not import zmq!",
                "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
                "If you did build pyzmq in-place, then this is a real error.",
            ]))
            sys.exit(1)

        info("Testing pyzmq-%s with libzmq-%s" %
             (zmq.pyzmq_version(), zmq.zmq_version()))
        p = Popen([
            sys.executable, '-m', 'pytest', '-v',
            os.path.join('zmq', 'tests')
        ])
        p.wait()
        sys.exit(p.returncode)
Beispiel #47
0
def directory_server(store: DirectoryServerStore, zmq_context: Context):
    # pylint: disable=no-member # These zmq.ROUTER and zmq.PUB must be actually exists
    print("Starting on libzmq {} with PyZMQ {}".format(zmq.zmq_version(),
                                                       zmq.pyzmq_version()))
    entrypoint: Socket = zmq_context.socket(zmq.ROUTER)
    entrypoint.bind("tcp://127.0.0.1:5350")  # This is just a PROTOTYPE!
    pub_file_changes: Socket = zmq_context.socket(zmq.PUB)
    pub_file_changes.bind("tcp://127.0.0.1:5351")
    poller = Poller()
    poller.register(entrypoint, flags=zmq.POLLIN)
    print(
        "Directory server is started on 127.0.0.1:5350 (commands) and 127.0.0.1:5351 (file_changes_push)"
    )
    while True:
        events: List[Tuple[Socket, int]] = poller.poll()
        for socket, _ in events:
            frames: List[Frame] = socket.recv_multipart(copy=False)
            id_frame: Frame = frames.pop(0)
            empty_frame: Frame = frames.pop(0)
            assert (len(empty_frame.bytes) == 0)
            command_frame: Frame = frames.pop(0)
            command = str(command_frame.bytes, encoding='utf8')
            if command == 'ping':
                ping_handler(store, socket, frames, id_frame)
            elif command == 'device.cast_address':
                casting_address_handler(store, socket, frames, id_frame)
            elif command == 'device.get_addresses':
                get_addresses_handler(store, socket, frames, id_frame)
            elif command == 'fs.list':
                file_list_handler(store, socket, id_frame)
            elif command == 'fs.declare':
                file_declare_handler(store, socket, frames, id_frame,
                                     pub_file_changes)
            elif command == 'fs.disown':
                file_disown_handler(store, socket, frames, id_frame,
                                    pub_file_changes)
            elif command == 'fs.get':
                file_get_handler(store, socket, frames, id_frame)
Beispiel #48
0
    def __init__(self):
        Tool.__init__(self)

        # * Initialize ZMQ context and open subscriber, publisher sockets
        self.logger.debug("ZMQ version: {}, PyZMQ version: {}".format(
            zmq.zmq_version(), zmq.pyzmq_version()))
        # ** Context
        self.zmqContext = zmq.Context()
        # ** Subscriber
        self.subSocket = self.zmqContext.socket(zmq.SUB)
        self.subSocket.connect(self.sub_address)
        time.sleep(0.005)  # mandatory sleep for ZMQ backend
        self.logger.debug("[sub] Connected to {}".format(self.sub_address))
        # ** Subscriber topics for input messages
        self.subSocket.setsockopt(zmq.SUBSCRIBE, self.sub_topic)
        self.subSocket.setsockopt(zmq.LINGER, self.sub_socket_linger)
        self.logger.debug("[sub]Subscribed to topic \"{}\"".format(
            self.sub_topic))
        time.sleep(0.005)  # mandatory sleep for ZMQ backend
        # ** Publisher
        self.pubSocket = self.zmqContext.socket(zmq.PUB)
        self.pubSocket.bind(self.pub_address)
        time.sleep(0.005)  # mandatory sleep for ZMQ backend
        self.logger.debug("[pub] Bound to {}".format(self.pub_address))

        # * Initialize other members
        self.valid = False
        self.buttons = [0, 0]  # primary, secondary
        self.transform = hm.translation(hm.identity(), self.position_offset)
        #self.position = self.position_offset
        self.loop = True  # TODO ensure this is properly shared across threads

        # * Start sensing loop
        self.senseThread = Thread(target=self.senseLoop)
        self.senseThread.daemon = True  # to prevent indefinite wait on recv()
        self.senseThread.start()
        time.sleep(0.005)  # sleep to allow child thread to run
Beispiel #49
0
def set_tcp_keepalive(sock, opts=None):
    # Warn if ZMQ < 3.2
    try:
        zmq_version_info = zmq.zmq_version_info()
    except AttributeError:
        # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
        # using zmq.zmq_version() and build a version info tuple.
        zmq_version_info = tuple(
            [int(x) for x in zmq.zmq_version().split('.')]
        )
    if zmq_version_info < (3, 2):
        log.warning(
            'You have a version of ZMQ less than ZMQ 3.2! There are '
            'known connection keep-alive issues with ZMQ < 3.2 which '
            'may result in loss of contact with minions. Please '
            'upgrade your ZMQ!'
        )

    if hasattr(zmq, 'TCP_KEEPALIVE') and opts:
        if 'tcp_keepalive' in opts:
            sock.setsockopt(
                zmq.TCP_KEEPALIVE, opts['tcp_keepalive']
            )
        if 'tcp_keepalive_idle' in opts:
            sock.setsockopt(
                zmq.TCP_KEEPALIVE_IDLE, opts['tcp_keepalive_idle']
            )
            if 'tcp_keepalive_cnt' in opts:
                sock.setsockopt(
                    zmq.TCP_KEEPALIVE_CNT, opts['tcp_keepalive_cnt']
                )
            if 'tcp_keepalive_intvl' in opts:
                sock.setsockopt(
                    zmq.TCP_KEEPALIVE_INTVL, opts['tcp_keepalive_intvl']
                )
    return sock
Beispiel #50
0
 def test_zmq_version(self):
     v = zmq.zmq_version()
     self.assertTrue(isinstance(v, str))
Beispiel #51
0
#-----------------------------------------------------------------------------

import os
import warnings

import zmq


if os.name == 'nt':
    if zmq.__version__ < '2.1.7':
        raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.7 on Windows, "
        "and you appear to have %s"%zmq.__version__)
elif zmq.__version__ < '2.1.4':
    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)

if zmq.zmq_version() >= '3.0.0':
    warnings.warn("""libzmq 3 detected.
    It is unlikely that IPython's zmq code will work properly.
    Please install libzmq stable, which is 2.1.x or 2.2.x""",
    RuntimeWarning)


from IPython.utils.pickleutil import Reference

from .client.asyncresult import *
from .client.client import Client
from .client.remotefunction import *
from .client.view import *
from .controller.dependency import *

    except Exception, e:
        logger.error(e)
    finally:
        logger.info("Exiting..")


if __name__ == '__main__':

    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', "--debug", action="store_true", help="Enable Debugging")
    parser.add_argument('-c', '--conf', required=True, help="JSON Config File")
    args = parser.parse_args()

    # configure logging
    logging.basicConfig(format="%(asctime)s [%(name)s:%(lineno)d][%(levelname)s] %(message)s", level=logging.INFO)

    # enable debugging
    if args.debug:
        logger.setLevel(logging.DEBUG)

    # open config and parse
    with open(args.conf) as data_file:
        conf = json.load(data_file)

    # log configuration
    logger.info('zmq python bindings %s, libzmq version %s' % (repr(zmq.__version__), zmq.zmq_version()))

    # run
    read(conf)
    import sys
    logger.info('Python Version: %s' % sys.version)
    logger.info('Platform: %s' % sys.platform)
except Exception:
    logger.error('Failed to find python version')

try:
    import sys
    logger.info('windows version: %s' % str(sys.getwindowsversion()))
except Exception:
    pass

try:
    import zmq
    logger.info('PyZMQ Version: %s' % zmq.__version__)
    logger.info('ZMQ Version: %s' % zmq.zmq_version())
except Exception:
    logger.error('Failed to find PyZMQ version')

try:
    import h5py
    logger.info('h5py Version: %s' % h5py.version.info)
except Exception:
    logger.error('Failed to find h5py version')

try:
    if 'PySide' in sys.modules.copy():
        import PySide
        logger.info('PySide Version: %s' % PySide.__version__)
        logger.info('Qt Version: %s' % PySide.QtCore.__version__)
    else:
Beispiel #54
0
#!/usr/bin/python
# -*- coding: latin-1 -*-

import socket
import threading
import time
from queue import Queue

import zmq

print("Current libzmq version is %s" % zmq.zmq_version())
print("Current  pyzmq version is %s" % zmq.__version__)


class Server(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self, args=(queue, ))
        self.ctx = zmq.Context()
        self.q = queue

    def setup_listening_socket(self):
        self.listener = self.ctx.socket(zmq.SUB)
        self.listener.setsockopt_string(zmq.SUBSCRIBE, "HERE")

    def setup_talking_socket(self):
        self.talker = self.ctx.socket(zmq.ROUTER)
        self.talker.bind('tcp://*:5556')

    def run(self):
        self.setup_listening_socket()
        self.setup_talking_socket()
Beispiel #55
0
    server.send(b"Hello")

    if client.poll(1000):
        msg = client.recv()
        if msg == b"Hello":
            logging.info("Ironhouse test OK")
    else:
        logging.error("Ironhouse test FAIL")

    # stop auth thread
    auth.stop()


if __name__ == '__main__':
    if zmq.zmq_version_info() < (4, 0):
        raise RuntimeError(
            "Security is not supported in libzmq version < 4.0. libzmq version {0}".format(
                zmq.zmq_version()
            )
        )

    if '-v' in sys.argv:
        level = logging.DEBUG
    else:
        level = logging.INFO

    logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")

    run()
            while (True):
                try:
                    msg = socket.recv_json(zmq.NOBLOCK)
                except zmq.error.Again:
                    break
                except Exception, e:
                    logging.info(e)
                    break
                else:
                    logging.info(pprint.pformat(msg))

    except Exception, e:
        logging.info(e)
    finally:
        raw_input('Press Enter to continue...')


if (__name__ == '__main__'):
    logging.info('zmq python bindings %s, libzmq version %s' %
                 (repr(zmq.__version__), zmq.zmq_version()))
    parser = argparse.ArgumentParser(
        description='Verbose QuakeLive server statistics')
    parser.add_argument('--host',
                        default=HOST,
                        help='ZMQ URI to connect to. Defaults to %s' % HOST)
    parser.add_argument('--password', required=False)
    args = parser.parse_args()
    #logging.debug( repr( args ) )
    verbose(args)
Beispiel #57
0
    client.curve_serverkey = server_public
    client.connect('tcp://127.0.0.1:9000')

    server.send(b"Hello")

    if client.poll(1000):
        msg = client.recv()
        if msg == b"Hello":
            logging.info("Ironhouse test OK")
    else:
        logging.error("Ironhouse test FAIL")

    # stop auth thread
    auth.stop()


if __name__ == '__main__':
    if zmq.zmq_version_info() < (4, 0):
        raise RuntimeError(
            "Security is not supported in libzmq version < 4.0. libzmq version {0}"
            .format(zmq.zmq_version()))

    if '-v' in sys.argv:
        level = logging.DEBUG
    else:
        level = logging.INFO

    logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")

    run()
Beispiel #58
0
    def run(self, ip, port, register=False):
        def _invalid(_):
            self._send_error('Unknown command')
            return False

        try:
            self.__load_keychain()
        except Exception as err:
            raise Exception('Unable to load issuer keychain')

        try:
            self.output('Launching CA listener')
            context = zmq.Context()
            self.output("Listening socket use ZMQ version {v}".format(v=zmq.zmq_version()), level="DEBUG")
            self._socket = context.socket(zmq.REP)
            self._socket.bind('tcp://{host}:{port}'.format(host=ip, port=port))
            self.output("Listener Socket bind to tcp://{host}:{port}".format(host=ip, port=port))
        except zmq.ZMQError as err:
            raise upkica.core.UPKIError(20,"Stalker process failed with: {e}".format(e=err))
        except Exception as err:
            raise upkica.core.UPKIError(20,"Error on connection: {e}".format(e=err))

        self._run = True

        while self._run:
            try:
                msg = self._socket.recv_json()
            except zmq.ZMQError as e:
                self.output('ZMQ Error: {err}'.format(err=e), level="ERROR")
                continue
            except ValueError:
                self.output('Received unparsable message', level="ERROR")
                continue
            except SystemExit:
                self.output('Poison listener...', level="WARNING")
                break
            
            try:
                self.output('Receive {task} action...'.format(task=msg['TASK']), level="INFO")
                self.output('Action message: {param}'.format(param=msg), level="DEBUG")
                task = "_upki_{t}".format(t=msg['TASK'].lower())
            except KeyError:
                self.output('Received invalid message', level="ERROR")
                continue

            try:
                params = msg['PARAMS']
            except KeyError:
                params = {}

            func = getattr(self, task, _invalid)
            
            try:
                res = func(params)
            except Exception as err:
                self.output('Error: {e}'.format(e=err), level='error')
                self._send_error(err)
                continue

            if res is False:
                continue

            try:
                self._send_answer(res)
            except Exception as err:
                self.output('Error: {e}'.format(e=err), level='error')
                self._send_error(err)
                continue