def test_unicode_sockopts(self):
"""test setting/getting sockopts with unicode strings"""
topic = "tést"
if str is not unicode:
topic = topic.decode('utf8')
p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
self.assertEquals(s.send_unicode, s.send_unicode)
self.assertEquals(p.recv_unicode, p.recv_unicode)
self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
if zmq.zmq_version() < '4.0.0':
self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
self.assertRaises(TypeError, s.getsockopt_unicode, zmq.SUBSCRIBE)
if zmq.zmq_version() >= '4.0.0':
# skip the rest on 4.0, because IDENTITY was removed
return
st = s.getsockopt(zmq.IDENTITY)
self.assertEquals(st.decode('utf16'), s.getsockopt_unicode(zmq.IDENTITY, 'utf16'))
time.sleep(0.1) # wait for connection/subscription
p.send_unicode(topic,zmq.SNDMORE)
p.send_unicode(topic*2, encoding='latin-1')
self.assertEquals(topic, s.recv_unicode())
self.assertEquals(topic*2, s.recv_unicode(encoding='latin-1'))
def test_single_socket_forwarder_bind(self):
if zmq.zmq_version() in ('4.1.1', '4.0.6'):
raise SkipTest("libzmq-%s broke single-socket devices" % zmq.zmq_version())
dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1)
# select random port:
binder = self.context.socket(zmq.REQ)
port = binder.bind_to_random_port('tcp://127.0.0.1')
binder.close()
time.sleep(0.1)
req = self.context.socket(zmq.REQ)
req.connect('tcp://127.0.0.1:%i'%port)
dev.bind_in('tcp://127.0.0.1:%i'%port)
dev.start()
time.sleep(.25)
msg = b'hello'
req.send(msg)
self.assertEqual(msg, self.recv(req))
del dev
req.close()
dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1)
# select random port:
binder = self.context.socket(zmq.REQ)
port = binder.bind_to_random_port('tcp://127.0.0.1')
binder.close()
time.sleep(0.1)
req = self.context.socket(zmq.REQ)
req.connect('tcp://127.0.0.1:%i'%port)
dev.bind_in('tcp://127.0.0.1:%i'%port)
dev.start()
time.sleep(.25)
msg = b'hello again'
req.send(msg)
self.assertEqual(msg, self.recv(req))
del dev
req.close()
def test_router_router(self):
"""test router-router MQ devices"""
if zmq.zmq_version() >= '4.0.0':
raise SkipTest("Only for libzmq < 4")
dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, asbytes('in'), asbytes('out'))
dev.setsockopt_in(zmq.LINGER, 0)
dev.setsockopt_out(zmq.LINGER, 0)
dev.setsockopt_mon(zmq.LINGER, 0)
binder = self.context.socket(zmq.DEALER)
porta = binder.bind_to_random_port('tcp://127.0.0.1')
portb = binder.bind_to_random_port('tcp://127.0.0.1')
binder.close()
time.sleep(0.1)
a = self.context.socket(zmq.DEALER)
a.identity = asbytes('a')
b = self.context.socket(zmq.DEALER)
b.identity = asbytes('b')
a.connect('tcp://127.0.0.1:%i'%porta)
dev.bind_in('tcp://127.0.0.1:%i'%porta)
b.connect('tcp://127.0.0.1:%i'%portb)
dev.bind_out('tcp://127.0.0.1:%i'%portb)
dev.start()
time.sleep(0.2)
msg = [ asbytes(m) for m in ('hello', 'there')]
a.send_multipart([asbytes('b')]+msg)
bmsg = self.recv_multipart(b)
self.assertEquals(bmsg, [asbytes('a')]+msg)
b.send_multipart(bmsg)
amsg = self.recv_multipart(a)
self.assertEquals(amsg, [asbytes('b')]+msg)
def __init__(self):
Tool.__init__(self)
# * Initialize ZMQ context and open subscriber, publisher sockets
self.logger.debug("ZMQ version: {}, PyZMQ version: {}".format(zmq.zmq_version(), zmq.pyzmq_version()))
# ** Context
self.zmqContext = zmq.Context()
# ** Subscriber
self.subSocket = self.zmqContext.socket(zmq.SUB)
self.subSocket.connect(self.sub_address)
time.sleep(0.005) # mandatory sleep for ZMQ backend
self.logger.debug("[sub] Connected to {}".format(self.sub_address))
# ** Subscriber topics for input messages
self.subSocket.setsockopt(zmq.SUBSCRIBE, self.sub_topic)
self.subSocket.setsockopt(zmq.LINGER, self.sub_socket_linger)
self.logger.debug("[sub]Subscribed to topic \"{}\"".format(self.sub_topic))
time.sleep(0.005) # mandatory sleep for ZMQ backend
# ** Publisher
self.pubSocket = self.zmqContext.socket(zmq.PUB)
self.pubSocket.bind(self.pub_address)
time.sleep(0.005) # mandatory sleep for ZMQ backend
self.logger.debug("[pub] Bound to {}".format(self.pub_address))
# * Initialize other members
self.valid = False
self.buttons = [0, 0] # primary, secondary
self.transform = hm.translation(hm.identity(), self.position_offset)
#self.position = self.position_offset
self.loop = True # TODO ensure this is properly shared across threads
# * Start sensing loop
self.senseThread = Thread(target=self.senseLoop)
self.senseThread.daemon = True # to prevent indefinite wait on recv()
self.senseThread.start()
time.sleep(0.005) # sleep to allow child thread to run
def test_labels(self):
"""test device support for SNDLABEL"""
if zmq.zmq_version() < '3.0.0':
raise SkipTest("Only for libzmq 3")
dev = devices.ThreadDevice(zmq.QUEUE, zmq.XREP, -1)
# select random port:
binder = self.context.socket(zmq.XREQ)
port = binder.bind_to_random_port('tcp://127.0.0.1')
binder.close()
time.sleep(0.1)
req = self.context.socket(zmq.REQ)
req.connect('tcp://127.0.0.1:%i'%port)
dev.bind_in('tcp://127.0.0.1:%i'%port)
dev.start()
time.sleep(.25)
msg = asbytes('hello')
req.send(msg, zmq.SNDLABEL)
req.send(msg, zmq.SNDMORE)
req.send(msg)
self.assertEquals(msg, self.recv(req))
self.assertTrue(req.rcvlabel)
self.assertEquals(msg, self.recv(req))
self.assertTrue(req.rcvmore)
self.assertEquals(msg, self.recv(req))
del dev
req.close()
def test_router_router(self):
"""test router-router MQ devices"""
if zmq.zmq_version() >= "4.0.0":
raise SkipTest("Only for libzmq < 4")
dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, "in", "out")
dev.setsockopt_in(zmq.LINGER, 0)
dev.setsockopt_out(zmq.LINGER, 0)
dev.setsockopt_mon(zmq.LINGER, 0)
binder = self.context.socket(zmq.DEALER)
porta = binder.bind_to_random_port("tcp://127.0.0.1")
portb = binder.bind_to_random_port("tcp://127.0.0.1")
binder.close()
time.sleep(0.1)
a = self.context.socket(zmq.DEALER)
a.identity = asbytes("a")
b = self.context.socket(zmq.DEALER)
b.identity = asbytes("b")
a.connect("tcp://127.0.0.1:%i" % porta)
dev.bind_in("tcp://127.0.0.1:%i" % porta)
b.connect("tcp://127.0.0.1:%i" % portb)
dev.bind_out("tcp://127.0.0.1:%i" % portb)
dev.start()
time.sleep(0.2)
msg = [asbytes(m) for m in ("hello", "there")]
a.send_multipart(["b"] + msg)
bmsg = self.recv_multipart(b)
self.assertEquals(bmsg, ["a"] + msg)
b.send_multipart(bmsg)
amsg = self.recv_multipart(a)
self.assertEquals(amsg, ["b"] + msg)
def test_prefix(self):
if zmq.zmq_version() < '3.0.0':
raise SkipTest("Only applies to libzmq >= 3.0")
xrep, xreq = self.create_bound_pair(zmq.XREP, zmq.XREQ)
msg = [ asbytes(p) for p in 'hi there'.split() ]
xreq.send_multipart(msg)
recvd = xrep.recv_multipart()
self.assertTrue(isinstance(recvd, tuple))
self.assertEquals(len(recvd), 2)
prefix, real = recvd
self.assertTrue(isinstance(prefix, list))
self.assertEquals(len(prefix), 1)
self.assertEquals(real, msg)
xrep.send_multipart(real, prefix=prefix)
echo = xreq.recv_multipart()
self.assertTrue(isinstance(echo, list))
self.assertEquals(echo, real)
extra = [asbytes('pre')]
xrep.send_multipart(msg, prefix=prefix+extra)
recvd = xreq.recv_multipart()
self.assertTrue(isinstance(recvd, tuple))
self.assertEquals(len(recvd), 2)
prefix, real = recvd
self.assertTrue(isinstance(prefix, list))
self.assertEquals(len(prefix), 1)
self.assertEquals(prefix, extra)
self.assertEquals(real, msg)
def check_for_zmq(minimum_version, module='IPython.zmq'):
min_vlist = [int(n) for n in minimum_version.split('.')]
try:
import zmq
except ImportError:
raise ImportError("%s requires pyzmq >= %s"%(module, minimum_version))
pyzmq_version = zmq.__version__
vlist = [int(n) for n in re.findall(r'\d+', pyzmq_version)]
if 'dev' not in pyzmq_version and vlist < min_vlist:
raise ImportError("%s requires pyzmq >= %s, but you have %s"%(
module, minimum_version, pyzmq_version))
# fix missing DEALER/ROUTER aliases in pyzmq < 2.1.9
if not hasattr(zmq, 'DEALER'):
zmq.DEALER = zmq.XREQ
if not hasattr(zmq, 'ROUTER'):
zmq.ROUTER = zmq.XREP
if zmq.zmq_version() >= '4.0.0':
warnings.warn("""libzmq 4 detected.
It is unlikely that IPython's zmq code will work properly.
Please install libzmq stable, which is 2.1.x or 2.2.x""",
RuntimeWarning)
def enable_monitor(self, events=None):
# The standard approach of binding and then connecting does not
# work in this specific case. The event loop does not properly
# detect messages on the inproc transport which means that event
# messages get missed.
# pyzmq's 'get_monitor_socket' method can't be used because this
# performs the actions in the wrong order for use with an event
# loop.
# For more information on this issue see:
# http://lists.zeromq.org/pipermail/zeromq-dev/2015-July/029181.html
if (zmq.zmq_version_info() < (4,) or
zmq.pyzmq_version_info() < (14, 4,)):
raise NotImplementedError(
"Socket monitor requires libzmq >= 4 and pyzmq >= 14.4, "
"have libzmq:{}, pyzmq:{}".format(
zmq.zmq_version(), zmq.pyzmq_version()))
if self._monitor is None:
addr = "inproc://monitor.s-{}".format(self._zmq_sock.FD)
events = events or zmq.EVENT_ALL
_, self._monitor = yield from create_zmq_connection(
lambda: _ZmqEventProtocol(self._loop, self._protocol),
zmq.PAIR, connect=addr, loop=self._loop)
# bind must come after connect
self._zmq_sock.monitor(addr, events)
yield from self._monitor.wait_ready
def _printheader(self, log):
"""Prints header to log file; inspired by that in GPAW."""
log(logo)
log('Amp: Atomistic Machine-learning Package')
log('Developed by Andrew Peterson, Alireza Khorshidi, and others,')
log('Brown University.')
log(' PI Website: http://brown.edu/go/catalyst')
log(' Official repository: http://bitbucket.org/andrewpeterson/amp')
log(' Official documentation: http://amp.readthedocs.org/')
log(' Citation:')
log(' Khorshidi & Peterson, Computer Physics Communications')
log(' doi:10.1016/j.cpc.2016.05.010 (2016)')
log('=' * 70)
log('User: %s' % getuser())
log('Hostname: %s' % gethostname())
log('Date: %s' % now(with_utc=True))
uname = platform.uname()
log('Architecture: %s' % uname[4])
log('PID: %s' % os.getpid())
log('Amp version: %s' % 'NOT NUMBERED YET.') # FIXME/ap. Look at GPAW
ampdirectory = os.path.dirname(os.path.abspath(__file__))
log('Amp directory: %s' % ampdirectory)
commithash, commitdate = get_git_commit(ampdirectory)
log(' Last commit: %s' % commithash)
log(' Last commit date: %s' % commitdate)
log('Python: v{0}.{1}.{2}: %s'.format(*sys.version_info[:3]) %
sys.executable)
log('ASE v%s: %s' % (aseversion, os.path.dirname(ase.__file__)))
log('NumPy v%s: %s' %
(np.version.version, os.path.dirname(np.__file__)))
# SciPy is not a strict dependency.
try:
import scipy
log('SciPy v%s: %s' %
(scipy.version.version, os.path.dirname(scipy.__file__)))
except ImportError:
log('SciPy: not available')
# ZMQ an pxssh are only necessary for parallel calculations.
try:
import zmq
log('ZMQ/PyZMQ v%s/v%s: %s' %
(zmq.zmq_version(), zmq.pyzmq_version(),
os.path.dirname(zmq.__file__)))
except ImportError:
log('ZMQ: not available')
try:
import pxssh
log('pxssh: %s' % os.path.dirname(pxssh.__file__))
except ImportError:
log('pxssh: Not available from pxssh.')
try:
from pexpect import pxssh
except ImportError:
log('pxssh: Not available from pexpect.')
else:
import pexpect
log('pxssh (via pexpect v%s): %s' %
(pexpect.__version__, pxssh.__file__))
log('=' * 70)
def run(self):
"""Run the test suite, with nose, or unittest if nose is unavailable"""
# crude check for inplace build:
try:
import zmq
except ImportError:
print_exc()
fatal(
"\n ".join(
[
"Could not import zmq!",
"You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
"If you did build pyzmq in-place, then this is a real error.",
]
)
)
sys.exit(1)
info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
if nose is None:
warn("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
return self.run_unittest()
else:
return self.run_nose()
def print_info():
log.info("Python version: %s, %s",
'.'.join((str(e) for e in sys.version_info)),
sys.executable)
log.info("zeromq version: %s", zmq.zmq_version())
log.info("pyzmq version: %s", zmq.pyzmq_version())
log.info("track version: %s", str(track_base.version_info))
def __init__(self, min_version, msg='Feature'):
global _zmq_version
if _zmq_version is None:
from zmq import zmq_version
_zmq_version = zmq_version()
self.msg = msg
self.min_version = min_version
self.version = _zmq_version
def zmqversion():
'''
Return the zeromq version
'''
# Provides:
# zmqversion
import zmq
return {'zmqversion': zmq.zmq_version()}
def log_versions(logger):
logger.info(
'Versions:\n'
' PyZMQ: %s\n'
' libzmq: %s\n'
' Tornado: %s\n'
' SockJS-Tornado: Not Available\n' # TODO find the verison
' SockJS-ZMQProxy: %s\n',
zmq.pyzmq_version(),
zmq.zmq_version(),
tornado_version,
version
)
def test_bad_send_recv(self):
s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP)
if zmq.zmq_version() != "2.1.8":
# this doesn't work on 2.1.8
for copy in (True, False):
self.assertRaisesErrno(zmq.EFSM, s1.recv, copy=copy)
self.assertRaisesErrno(zmq.EFSM, s2.send, b"asdf", copy=copy)
# I have to have this or we die on an Abort trap.
msg1 = b"asdf"
msg2 = self.ping_pong(s1, s2, msg1)
self.assertEqual(msg1, msg2)
def test_router_dealer(self):
if zmq.zmq_version() >= '3.0.0':
raise SkipTest("Known bug in libzmq 3.0.0, see https://zeromq.jira.com/browse/LIBZMQ-232")
router, dealer = self.create_bound_pair(zmq.ROUTER, zmq.DEALER)
msg1 = asbytes('message1')
dealer.send(msg1)
ident = router.recv()
more = router.rcvmore()
self.assertEquals(more, True)
msg2 = router.recv()
self.assertEquals(msg1, msg2)
more = router.rcvmore()
self.assertEquals(more, False)
def test_router_dealer(self):
if zmq.zmq_version() >= '4.0.0':
raise SkipTest("ROUTER/DEALER change in 4.0")
router, dealer = self.create_bound_pair(zmq.ROUTER, zmq.DEALER)
msg1 = asbytes('message1')
dealer.send(msg1)
ident = self.recv(router)
more = router.rcvmore
self.assertEquals(more, True)
msg2 = self.recv(router)
self.assertEquals(msg1, msg2)
more = router.rcvmore
self.assertEquals(more, False)
def test_int_sockopts(self):
"test non-uint64 sockopts"
v = zmq.zmq_version()
if not v >= '2.1':
raise SkipTest("only on libzmq >= 2.1")
elif v < '3.0':
hwm = zmq.HWM
default_hwm = 0
else:
hwm = zmq.SNDHWM
default_hwm = 1000
p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
p.setsockopt(zmq.LINGER, 0)
self.assertEquals(p.getsockopt(zmq.LINGER), 0)
p.setsockopt(zmq.LINGER, -1)
self.assertEquals(p.getsockopt(zmq.LINGER), -1)
self.assertEquals(p.getsockopt(hwm), default_hwm)
p.setsockopt(hwm, 11)
self.assertEquals(p.getsockopt(hwm), 11)
# p.setsockopt(zmq.EVENTS, zmq.POLLIN)
self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
self.assertRaisesErrno(zmq.EINVAL, p.setsockopt,zmq.EVENTS, 2**7-1)
self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
# check for overflow / wrong type:
errors = []
backref = {}
constants = zmq.core.constants
for name in constants.__all__:
value = getattr(constants, name)
if isinstance(value, int):
backref[value] = name
for opt in zmq.core.constants.int_sockopts+zmq.core.constants.int64_sockopts:
sopt = backref[opt]
try:
n = p.getsockopt(opt)
except zmq.ZMQError:
e = sys.exc_info()[1]
errors.append("getsockopt(zmq.%s) raised '%s'."%(sopt, e))
else:
if n > 2**31:
errors.append("getsockopt(zmq.%s) returned a ridiculous value."
" It is probably the wrong type."%sopt)
if errors:
self.fail('\n'.join(errors))
def test_closing_full():
ctx = zmq.Context()
ver = zmq.zmq_version()[:3]
global port
out = ctx.socket( zmq.XREQ )
out.bind( "tcp://*:%d" % port )
inc = ctx.socket( zmq.XREP )
inc.connect( "tcp://localhost:%d" % port )
port += 1
try:
# Test that ..._multipart and zmq.RCVMORE behave as we think
out.send_multipart( ['', 'TEST1'] )
msg = inc.recv_multipart()
if ver in ("2.1", "3.1"):
assert len( msg ) == 3
elif ver in ("3.0"):
assert isinstance( msg, tuple )
assert len( msg[0] ) == 1
assert len( msg[1] ) == 2
else:
assert False and "Unknown zmq version"
out.send_multipart( ['', 'TEST2'] )
msg = []
poller = zmq.Poller()
poller.register(inc, zmq.POLLIN)
while poller.poll( timeout=100 ):
msg.append( inc.recv() )
if not inc.getsockopt( zmq.RCVMORE ):
break
assert len( msg ) == 3
## Add another stage; see if we can intercept and re-route between
## inc/nxt
#nxt = ctx.socket( zmq.XREQ )
#nxt.connect( "tcp://localhost:%d" % port )
#
#lst = ctx.socket( zme.XREP )
#nxt.bind( "tcp://*:%d" % port )
#port += 1
finally:
out.close()
inc.close()
ctx.term()
def run(self):
"""Run the test suite with py.test"""
# crude check for inplace build:
try:
import zmq
except ImportError:
print_exc()
fatal('\n '.join(["Could not import zmq!",
"You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
"If you did build pyzmq in-place, then this is a real error."]))
sys.exit(1)
info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
p = Popen([sys.executable, '-m', 'pytest', '-v', os.path.join('zmq', 'tests')])
p.wait()
sys.exit(p.returncode)
def check_for_zmq(minimum_version, module='IPython.zmq'):
try:
import zmq
except ImportError:
raise ImportError("%s requires pyzmq >= %s"%(module, minimum_version))
pyzmq_version = zmq.__version__
if 'dev' not in pyzmq_version and V(pyzmq_version) < V(minimum_version):
raise ImportError("%s requires pyzmq >= %s, but you have %s"%(
module, minimum_version, pyzmq_version))
if V(zmq.zmq_version()) >= V('4.0.0'):
warnings.warn("""libzmq 4 detected.
It is unlikely that IPython's zmq code will work properly.
Please install libzmq stable, which is 2.1.x or 2.2.x""",
RuntimeWarning)
def level_1(info):
# print header (some general info)
info += ["name: %s" % self.job_name]
info += ["version: %s" % getattr(self, "__version__", "")]
info += ["service: %s" % self.service_name]
info += ["pid: %i" % os.getpid()]
info += ["started: %s" % strftime(getattr(self, "started", ""))]
info += ["status: %s" % getattr(self, "status", "")]
info += ["time-consumed: %s" % " ".join(map(str, os.times()[:4]))]
info += ["info-period: %s" % self.stats_period]
info += ["info-written: %s" % strftime(time.time())]
info += ["platform: %s" % platform.platform()]
info += ["python: %s" % platform.python_version()]
info += ["skytools: %s" % skytools.__version__]
info += ["libzmq: %s" % zmq.zmq_version()]
info += ["pyzmq: %s" % zmq.pyzmq_version()]
info += [""]
def test_2_1_sockopts(self):
"test non-uint64 sockopts introduced in zeromq 2.1.0"
v = list(map(int, zmq.zmq_version().split('.', 2)[:2]))
if not (v[0] >= 2 and v[1] >= 1):
raise SkipTest
p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
p.setsockopt(zmq.LINGER, 0)
self.assertEquals(p.getsockopt(zmq.LINGER), 0)
p.setsockopt(zmq.LINGER, -1)
self.assertEquals(p.getsockopt(zmq.LINGER), -1)
# p.setsockopt(zmq.EVENTS, zmq.POLLIN)
self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
self.assertRaisesErrno(zmq.EINVAL, p.setsockopt,zmq.EVENTS, 2**7-1)
self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
def set_tcp_keepalive(sock, opts=None):
# Warn if ZMQ < 3.2
try:
zmq_version_info = zmq.zmq_version_info()
except AttributeError:
# PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
# using zmq.zmq_version() and build a version info tuple.
zmq_version_info = tuple(
[int(x) for x in zmq.zmq_version().split('.')]
)
if zmq_version_info < (3, 2):
log.warning(
'You have a version of ZMQ less than ZMQ 3.2! There are '
'known connection keep-alive issues with ZMQ < 3.2 which '
'may result in loss of contact with minions. Please '
'upgrade your ZMQ!'
)
####
# try:
# ip=socket.gethostbyname(socket.gethostname())
# if ip == "127.0.0.1":
# raise Exception("127.0.0.1")
# except Exception,e:
# (status,ip) = commands.getstatusoutput("/sbin/ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d 'addr:'")
# ip = ip
if opts['ha'] == True:
while True:
time.sleep(5)
try:
sys.path.append("/usr/lib/python2.7/site-packages/salt")
from zkclient import ZKClient, zookeeper, watchmethod
zk = ZKClient('172.16.163.14:2181',1000)
if zk.exists('/broker'):
value = zk.get('/broker')
logging.error(str(value[0]))
if value[0] == opts['id']:
break
else:
zk.create('/broker')
zk.set('/broker',data='%s'%opts['id'])
logging.error("create the zk path is successful !")
continue
except Exception,e:
logging.error(str(e))
continue
def setup(self):
sock = self.socket
sock.identity = identity = str(uuid.uuid4())
_log.debug("ROUTER SOCK identity: {}".format(sock.identity))
if self._monitor:
Monitor(sock.get_monitor_socket()).start()
sock.bind('inproc://vip')
_log.debug('In-process VIP router bound to inproc://vip')
sock.zap_domain = 'vip'
addr = self.local_address
if not addr.identity:
addr.identity = identity
if not addr.domain:
addr.domain = 'vip'
addr.server = 'CURVE'
addr.secretkey = self._secretkey
addr.bind(sock)
_log.debug('Local VIP router bound to %s' % addr)
for address in self.addresses:
if not address.identity:
address.identity = identity
if (address.secretkey is None
and address.server not in ['NULL', 'PLAIN']
and self._secretkey):
address.server = 'CURVE'
address.secretkey = self._secretkey
if not address.domain:
address.domain = 'vip'
address.bind(sock)
_log.debug('Additional VIP router bound to %s' % address)
self._ext_routing = None
self._ext_routing = RoutingService(self.socket, self.context,
self._socket_class, self._poller,
self._addr, self._instance_name)
self._pubsub = PubSubService(self.socket, self._protected_topics,
self._ext_routing)
self._ext_rpc = ExternalRPCService(self.socket, self._ext_routing)
self._poller.register(sock, zmq.POLLIN)
_log.debug("ZMQ version: {}".format(zmq.zmq_version()))
def get_monitor_socket(self, events=None, addr=None):
"""Return a connected PAIR socket ready to receive the event notifications.
.. versionadded:: libzmq-4.0
.. versionadded:: 14.0
Parameters
----------
events : int [default: ZMQ_EVENTS_ALL]
The bitmask defining which events are wanted.
addr : string [default: None]
The optional endpoint for the monitoring sockets.
Returns
-------
socket : (PAIR)
The socket is already connected and ready to receive messages.
"""
# safe-guard, method only available on libzmq >= 4
if zmq.zmq_version_info() < (4, ):
raise NotImplementedError(
"get_monitor_socket requires libzmq >= 4, have %s" %
zmq.zmq_version())
# if already monitoring, return existing socket
if self._monitor_socket:
if self._monitor_socket.closed:
self._monitor_socket = None
else:
return self._monitor_socket
if addr is None:
# create endpoint name from internal fd
addr = "inproc://monitor.s-%d" % self.FD
if events is None:
# use all events
events = zmq.EVENT_ALL
# attach monitoring socket
self.monitor(addr, events)
# create new PAIR socket and connect it
self._monitor_socket = self.context.socket(zmq.PAIR)
self._monitor_socket.connect(addr)
return self._monitor_socket
def __init__(self, opts):
'''
Create a salt broker instance
'''
self.opts = opts
# Warn if ZMQ < 3.2
try:
zmq_version_info = zmq.zmq_version_info()
except AttributeError:
# PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
# using zmq.zmq_version() and build a version info tuple.
zmq_version_info = tuple(
[int(x) for x in zmq.zmq_version().split('.')])
if zmq_version_info < (3, 2):
log.warning(
'You have a version of ZMQ less than ZMQ 3.2! There are '
'known connection keep-alive issues with ZMQ < 3.2 which '
'may result in loss of contact with minions. Please '
'upgrade your ZMQ!')
def __init__(self, args):
super().__init__()
self.logger = set_logger(colored('VENTILATOR', 'magenta'),
args.verbose)
self.model_dir = args.model_dir
self.spm_model_file = args.spm_model_file
self.max_seq_len = args.max_seq_len
self.num_worker = args.num_worker
self.max_batch_size = args.max_batch_size
self.num_concurrent_socket = max(
8, args.num_worker * 2) # optimize concurrency for multi-clients
self.port = args.port
self.args = args
self.status_args = {
k: (v if k != 'pooling_strategy' else v.value)
for k, v in sorted(vars(args).items())
}
self.status_static = {
'tensorflow_version': _tf_ver_,
'python_version': sys.version,
'server_version': __version__,
'pyzmq_version': zmq.pyzmq_version(),
'zmq_version': zmq.zmq_version(),
'server_start_time': str(datetime.now()),
}
self.processes = []
self.logger.info(
'freeze, optimize and export graph, could take a while...')
with Pool(processes=1) as pool:
# optimize the graph, must be done in another process
from .graph import optimize_graph
self.graph_path, self.bert_config = pool.apply(
optimize_graph, (self.args, ))
# from .graph import optimize_graph
# self.graph_path = optimize_graph(self.args, self.logger)
if self.graph_path:
self.logger.info('optimized graph is stored at: %s' %
self.graph_path)
else:
raise FileNotFoundError(
'graph optimization fails and returns empty result')
self.is_ready = threading.Event()
def main():
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.copy_threshold = 0
socket.connect("tcp://localhost:50300")
print("using zmq version", zmq.__version__)
print("using libzmq version", zmq.zmq_version())
message = b"World"
print("Send: ", message)
res = socket.send(message, copy=False, track=True)
if res.done:
print("res: done")
else:
print("res: waiting")
res.wait()
print("res: waiting...")
def __init__(self, port=5558, port_out=5559, n_workers=1, verbose=False,
max_batch_size=32, task='coref'):
super().__init__()
self.logger = set_logger(colored('VENTILATOR', 'magenta'), verbose)
self.port = port
self.port_out = port_out
self.processes = []
self.is_ready = threading.Event()
self.n_workers = n_workers
self.n_concurrent_sockets = max(8, n_workers * 2)
self.max_batch_size = max_batch_size
self.status_static = {
'python_version': sys.version,
'server_version': __version__,
'pyzmq_version': zmq.pyzmq_version(),
'zmq_version': zmq.zmq_version(),
'server_start_time': str(datetime.now()),
}
self.Worker = WorkerRegistry[task]
def run(self):
"""Run the test suite, with nose, or unittest if nose is unavailable"""
# crude check for inplace build:
try:
import zmq
except ImportError:
print_exc()
fatal('\n '.join(["Could not import zmq!",
"You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
"If you did build pyzmq in-place, then this is a real error."]))
sys.exit(1)
info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
if nose is None:
warn("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
return self.run_unittest()
else:
return self.run_nose()
def get_full_version() -> Optional[Tuple[Dict, Dict]]:
"""
Get the version of libraries used in Jina and environment variables.
:return: Version information and environment variables
"""
from . import __version__, __proto_version__, __jina_env__
from google.protobuf.internal import api_implementation
import os, zmq, numpy, google.protobuf, grpc, yaml
from grpc import _grpcio_metadata
from pkg_resources import resource_filename
import platform
from .logging import default_logger
try:
info = {
'jina': __version__,
'jina-proto': __proto_version__,
'jina-vcs-tag': os.environ.get('JINA_VCS_VERSION', '(unset)'),
'libzmq': zmq.zmq_version(),
'pyzmq': numpy.__version__,
'protobuf': google.protobuf.__version__,
'proto-backend': api_implementation._default_implementation_type,
'grpcio': getattr(grpc, '__version__',
_grpcio_metadata.__version__),
'pyyaml': yaml.__version__,
'python': platform.python_version(),
'platform': platform.system(),
'platform-release': platform.release(),
'platform-version': platform.version(),
'architecture': platform.machine(),
'processor': platform.processor(),
'jina-resources': resource_filename('jina', 'resources'),
}
env_info = {k: os.getenv(k, '(unset)') for k in __jina_env__}
full_version = info, env_info
except Exception as e:
default_logger.error(str(e))
full_version = None
return full_version
def connect(server="localhost", port=5555):
"""
Connect to a running arkouda server.
Parameters
----------
server : str, optional
The hostname of the server (must be visible to the current
machine). Defaults to `localhost`.
port : int, optional
The port of the server. Defaults to 5555.
Returns
-------
None
On success, prints ``connected to tcp://<hostname>:<port>``
"""
global verbose, context, socket, pspStr, serverPid, connected
if connected == False:
print(zmq.zmq_version())
# "protocol://server:port"
pspStr = "tcp://{}:{}".format(server, port)
print("psp = ", pspStr)
# setup connection to arkouda server
context = zmq.Context()
socket = context.socket(zmq.REQ) # request end of the zmq connection
socket.connect(pspStr)
connected = True
#send the connect message
message = "connect"
if verbose: print("[Python] Sending request: %s" % message)
socket.send_string(message)
# get the response that the server has started
message = socket.recv_string()
if verbose: print("[Python] Received response: %s" % message)
print("connected to {}".format(pspStr))
def setup(self):
sock = self.socket
sock.identity = identity = str(uuid.uuid4())
_log.debug("ROUTER SOCK identity: {}".format(sock.identity))
if self._monitor:
Monitor(sock.get_monitor_socket()).start()
sock.bind('inproc://vip')
_log.debug('In-process VIP router bound to inproc://vip')
sock.zap_domain = 'vip'
addr = self.local_address
if not addr.identity:
addr.identity = identity
if not addr.domain:
addr.domain = 'vip'
addr.server = 'CURVE'
addr.secretkey = self._secretkey
addr.bind(sock)
_log.debug('Local VIP router bound to %s' % addr)
for address in self.addresses:
if not address.identity:
address.identity = identity
if (address.secretkey is None and
address.server not in ['NULL', 'PLAIN'] and
self._secretkey):
address.server = 'CURVE'
address.secretkey = self._secretkey
if not address.domain:
address.domain = 'vip'
address.bind(sock)
_log.debug('Additional VIP router bound to %s' % address)
self._ext_routing = None
self._ext_routing = RoutingService(self.socket, self.context,
self._socket_class, self._poller,
self._addr, self._instance_name)
self._pubsub = PubSubService(self.socket, self._protected_topics, self._ext_routing)
self._ext_rpc = ExternalRPCService(self.socket, self._ext_routing)
self._poller.register(sock, zmq.POLLIN)
_log.debug("ZMQ version: {}".format(zmq.zmq_version()))
def test_2_1_sockopts(self):
"test non-uint64 sockopts introduced in zeromq 2.1.0"
v = list(map(int, zmq.zmq_version().split('.', 2)[:2]))
if not (v[0] >= 2 and v[1] >= 1):
raise SkipTest
p, s = self.create_bound_pair(zmq.PUB, zmq.SUB)
p.setsockopt(zmq.LINGER, 0)
self.assertEquals(p.getsockopt(zmq.LINGER), 0)
p.setsockopt(zmq.LINGER, -1)
self.assertEquals(p.getsockopt(zmq.LINGER), -1)
self.assertEquals(p.getsockopt(zmq.HWM), 0)
p.setsockopt(zmq.HWM, 11)
self.assertEquals(p.getsockopt(zmq.HWM), 11)
# p.setsockopt(zmq.EVENTS, zmq.POLLIN)
self.assertEquals(p.getsockopt(zmq.EVENTS), zmq.POLLOUT)
self.assertRaisesErrno(zmq.EINVAL, p.setsockopt, zmq.EVENTS, 2**7 - 1)
self.assertEquals(p.getsockopt(zmq.TYPE), p.socket_type)
self.assertEquals(p.getsockopt(zmq.TYPE), zmq.PUB)
self.assertEquals(s.getsockopt(zmq.TYPE), s.socket_type)
self.assertEquals(s.getsockopt(zmq.TYPE), zmq.SUB)
def __init__(self, args):
self.args = args
self.logger = set_logger(self.__class__.__name__, self.args.verbose)
self.ctx = zmq.Context()
self.ctx.setsockopt(zmq.LINGER, 0)
self.logger.info('current libzmq version is %s, pyzmq version is %s' %
(zmq.zmq_version(), zmq.__version__))
self.receiver, recv_addr = build_socket(self.ctx, self.args.host_in,
self.args.port_in,
self.args.socket_in,
self.args.identity)
self.sender, send_addr = build_socket(self.ctx, self.args.host_out,
self.args.port_out,
self.args.socket_out,
self.args.identity)
self.logger.info(
'input %s:%s\t output %s:%s' %
(self.args.host_in, colored(self.args.port_in, 'yellow'),
self.args.host_out, colored(self.args.port_out, 'yellow')))
def monitor(self, addr, events=-1):
"""s.monitor(addr, flags)
Start publishing socket events on inproc.
See libzmq docs for zmq_monitor for details.
Note: requires libzmq >= 3.2
Parameters
----------
addr : str
The inproc url used for monitoring.
events : int [default: zmq.EVENT_ALL]
The zmq event bitmask for which events will be sent to the monitor.
"""
if zmq.zmq_version_info() < (3,2):
raise NotImplementedError("monitor requires libzmq >= 3.2, have %s" % zmq.zmq_version())
if events < 0:
events = zmq.EVENT_ALL
rc = C.zmq_socket_monitor(self._zmq_socket, addr, events)
def log_sysinfo(app: Flask, config: Config):
app.logger.info("ZMQ:")
app.logger.info(" zmq version: %s", zmq.zmq_version())
app.logger.info(" pyzmq version: %s", zmq.pyzmq_version())
app.logger.info(" zmq includes: %s", zmq.get_includes())
app.logger.info(" zmq library dirs: %s", zmq.get_library_dirs())
app.logger.info(" has: %s", [c for c in ZMQ_CAPABILITIES if zmq.has(c)])
app.logger.info("socket:")
app.logger.info(" fqdn: %s", socket.getfqdn())
app.logger.info(" has_ipv6: %s", socket.has_ipv6)
app.logger.info(" hostname: %s", socket.gethostname())
app.logger.info(" interfaces: %s", [i[1] for i in socket.if_nameindex()])
app.logger.info("os:")
app.logger.info(" ctermid: %s", os.ctermid())
app.logger.info(" cwd: %s", os.getcwd())
app.logger.info(" groups: %s", os.getgroups())
app.logger.info(" pgid: %d", os.getpgid(0))
app.logger.info(" pgrp: %d", os.getpgrp())
app.logger.info(" pid: %d", os.getpid())
app.logger.info(" ppid: %d", os.getppid())
app.logger.info(" priority_process: %d",
os.getpriority(os.PRIO_PROCESS, 0))
app.logger.info(" priority_pgrp: %d", os.getpriority(os.PRIO_PGRP, 0))
app.logger.info(" priority_user: %d", os.getpriority(os.PRIO_USER, 0))
app.logger.info(" resuid: ruid=%d, euid=%d, suid=%d", *os.getresuid())
app.logger.info(" resgid: rgid=%d, egid=%d, sgid=%d", *os.getresgid())
app.logger.info(" sid: %d", os.getsid(0))
app.logger.info(" supports_bytes_environ: %s", os.supports_bytes_environ)
app.logger.info(" uname: %s", os.uname())
app.logger.info(" cpu_count: %d", os.cpu_count())
app.logger.info("platform:")
app.logger.info(" %s", platform.platform())
app.logger.info(" python_build: %s", platform.python_build())
app.logger.info(" python_compiler: %s", platform.python_compiler())
app.logger.info(" python_branch: %s", platform.python_branch())
app.logger.info(" python_implementation: %s",
platform.python_implementation())
app.logger.info(" python_revision: %s", platform.python_revision())
app.logger.info(" python_version: %s", platform.python_version())
app.logger.info("getpass:"******" user: %s", getpass.getuser())
def __init__(self, args, hardprocesser=WKRHardWorker):
super().__init__()
self.hardprocessor_skeleton = hardprocesser
if not issubclass(self.hardprocessor_skeleton, WKRHardWorker):
raise AssertionError(
'hardprocesser must inherit from class WKRHardWorker')
self.model_dir = args.model_dir
self.num_worker = args.num_worker
self.device_map = args.device_map
self.gpu_memory_fraction = args.gpu_memory_fraction
self.all_cpu = args.cpu
self.num_concurrent_postsocket = max(8, args.num_worker * 2)
self.batch_size = args.batch_size
self.total_concurrent_socket = self.num_concurrent_postsocket
self.port = args.port
self.args = args
self.transfer_protocol = args.protocol
self.status_args = {k: v for k, v in sorted(vars(args).items())}
self.status_static = {
'python_version': sys.version,
'server_version': __version__,
'pyzmq_version': zmq.pyzmq_version(),
'zmq_version': zmq.zmq_version(),
'server_start_time': str(datetime.now()),
}
self.processes = []
self.logdir = args.log_dir
self.logger = set_logger(colored('NAVIGATOR', 'red'),
logger_dir=self.logdir,
verbose=args.verbose)
self.logger.info(
'freeze, optimize and export graph, could take a while...')
self.is_ready = threading.Event()
def build_device(self, mon_sub=asbytes(""), in_prefix=asbytes('in'), out_prefix=asbytes('out')):
if zmq.zmq_version() >= '3':
raise SkipTest("MonitoredQueues don't work reliably on libzmq >= 3.0.0")
self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB,
in_prefix, out_prefix)
alice = self.context.socket(zmq.PAIR)
bob = self.context.socket(zmq.PAIR)
mon = self.context.socket(zmq.SUB)
aport = alice.bind_to_random_port('tcp://127.0.0.1')
bport = bob.bind_to_random_port('tcp://127.0.0.1')
mport = mon.bind_to_random_port('tcp://127.0.0.1')
mon.setsockopt(zmq.SUBSCRIBE, mon_sub)
self.device.connect_in("tcp://127.0.0.1:%i"%aport)
self.device.connect_out("tcp://127.0.0.1:%i"%bport)
self.device.connect_mon("tcp://127.0.0.1:%i"%mport)
time.sleep(.2)
self.device.start()
self.sockets.extend([alice, bob, mon])
return alice, bob, mon
def _get_master_uri(master_ip, master_port, source_ip=None, source_port=None):
'''
Return the ZeroMQ URI to connect the Minion to the Master.
It supports different source IP / port, given the ZeroMQ syntax:
// Connecting using a IP address and bind to an IP address
rc = zmq_connect(socket, "tcp://192.168.1.17:5555;192.168.1.1:5555"); assert (rc == 0);
Source: http://api.zeromq.org/4-1:zmq-tcp
'''
libzmq_version_tup = tuple(map(int, zmq.zmq_version().split('.')))
pyzmq_version_tup = tuple(map(int, zmq.pyzmq_version().split('.')))
if libzmq_version_tup >= (4, 1, 6) and pyzmq_version_tup >= (16, 0, 1):
# The source:port syntax for ZeroMQ has been added in libzmq 4.1.6
# which is included in the pyzmq wheels starting with 16.0.1.
if source_ip or source_port:
if source_ip and source_port:
return 'tcp://{source_ip}:{source_port};{master_ip}:{master_port}'.format(
source_ip=source_ip,
source_port=source_port,
master_ip=master_ip,
master_port=master_port)
elif source_ip and not source_port:
return 'tcp://{source_ip}:0;{master_ip}:{master_port}'.format(
source_ip=source_ip,
master_ip=master_ip,
master_port=master_port)
elif not source_ip and source_port:
return 'tcp://0.0.0.0:{source_port};{master_ip}:{master_port}'.format(
source_port=source_port,
master_ip=master_ip,
master_port=master_port)
if source_ip or source_port:
log.warning(
'Unable to connect to the Master using a specific source IP / port'
)
log.warning(
'Consider upgrading to pyzmq >= 16.0.1 and libzmq >= 4.1.6')
return 'tcp://{master_ip}:{master_port}'.format(master_ip=master_ip,
master_port=master_port)
def package_version(package_name, destination):
"""Returns the package version string
Args:
package_name (str): the package name to look up
destination (list): the list to store the result (tuple) to
"""
try:
version = metadata_version(package_name)
except PackageNotFoundError:
version = "<unavailable>"
if package_name == "pyzmq":
from zmq import zmq_version
version = "{} (libzmq version {})".format(
version,
zmq_version(),
)
destination.append((package_name, version))
def __init__(self, args):
super().__init__()
self.logger = set_logger(colored('VENTILATOR', 'magenta'), args.verbose)
# self.model_dir = args.model_dir
# self.max_seq_len = args.max_seq_len
self.num_worker = args.num_worker
self.max_batch_size = args.max_batch_size
self.num_concurrent_socket = max(8, args.num_worker * 2) # optimize concurrency for multi-clients
self.port = args.port
self.args = args
self.status_args = {k: (v if k != 'pooling_strategy' else v.value) for k, v in sorted(vars(args).items())}
self.status_static = {
'python_version': sys.version,
'pyzmq_version': zmq.pyzmq_version(),
'zmq_version': zmq.zmq_version(),
'server_start_time': str(datetime.now()),
}
self.processes = []
self.logger.info('freeze, optimize and export graph, could take a while...')
self.is_ready = threading.Event()
def __init__(self, args):
super().__init__()
self.logger = set_logger(colored('VENTILATOR', 'magenta'))
self.num_worker = args.num_worker
self.max_batch_size = args.max_batch_size
self.num_concurrent_socket = max(
8, args.num_worker * 2) # optimize concurrency for multi-clients
self.port = args.port
self.args = args
self.status_args = {
k: (v if k != 'pooling_strategy' else v.value)
for k, v in sorted(vars(args).items())
}
self.status_static = {
'python_version': sys.version,
'server_version': __version__,
'pyzmq_version': zmq.pyzmq_version(),
'zmq_version': zmq.zmq_version(),
'server_start_time': str(datetime.now()),
}
self.processes = []
def run(self):
"""Run the test suite with py.test"""
# crude check for inplace build:
try:
import zmq
except ImportError:
print_exc()
fatal('\n '.join([
"Could not import zmq!",
"You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
"If you did build pyzmq in-place, then this is a real error.",
]))
sys.exit(1)
info("Testing pyzmq-%s with libzmq-%s" %
(zmq.pyzmq_version(), zmq.zmq_version()))
p = Popen([
sys.executable, '-m', 'pytest', '-v',
os.path.join('zmq', 'tests')
])
p.wait()
sys.exit(p.returncode)
def directory_server(store: DirectoryServerStore, zmq_context: Context):
# pylint: disable=no-member # These zmq.ROUTER and zmq.PUB must be actually exists
print("Starting on libzmq {} with PyZMQ {}".format(zmq.zmq_version(),
zmq.pyzmq_version()))
entrypoint: Socket = zmq_context.socket(zmq.ROUTER)
entrypoint.bind("tcp://127.0.0.1:5350") # This is just a PROTOTYPE!
pub_file_changes: Socket = zmq_context.socket(zmq.PUB)
pub_file_changes.bind("tcp://127.0.0.1:5351")
poller = Poller()
poller.register(entrypoint, flags=zmq.POLLIN)
print(
"Directory server is started on 127.0.0.1:5350 (commands) and 127.0.0.1:5351 (file_changes_push)"
)
while True:
events: List[Tuple[Socket, int]] = poller.poll()
for socket, _ in events:
frames: List[Frame] = socket.recv_multipart(copy=False)
id_frame: Frame = frames.pop(0)
empty_frame: Frame = frames.pop(0)
assert (len(empty_frame.bytes) == 0)
command_frame: Frame = frames.pop(0)
command = str(command_frame.bytes, encoding='utf8')
if command == 'ping':
ping_handler(store, socket, frames, id_frame)
elif command == 'device.cast_address':
casting_address_handler(store, socket, frames, id_frame)
elif command == 'device.get_addresses':
get_addresses_handler(store, socket, frames, id_frame)
elif command == 'fs.list':
file_list_handler(store, socket, id_frame)
elif command == 'fs.declare':
file_declare_handler(store, socket, frames, id_frame,
pub_file_changes)
elif command == 'fs.disown':
file_disown_handler(store, socket, frames, id_frame,
pub_file_changes)
elif command == 'fs.get':
file_get_handler(store, socket, frames, id_frame)
def __init__(self):
Tool.__init__(self)
# * Initialize ZMQ context and open subscriber, publisher sockets
self.logger.debug("ZMQ version: {}, PyZMQ version: {}".format(
zmq.zmq_version(), zmq.pyzmq_version()))
# ** Context
self.zmqContext = zmq.Context()
# ** Subscriber
self.subSocket = self.zmqContext.socket(zmq.SUB)
self.subSocket.connect(self.sub_address)
time.sleep(0.005) # mandatory sleep for ZMQ backend
self.logger.debug("[sub] Connected to {}".format(self.sub_address))
# ** Subscriber topics for input messages
self.subSocket.setsockopt(zmq.SUBSCRIBE, self.sub_topic)
self.subSocket.setsockopt(zmq.LINGER, self.sub_socket_linger)
self.logger.debug("[sub]Subscribed to topic \"{}\"".format(
self.sub_topic))
time.sleep(0.005) # mandatory sleep for ZMQ backend
# ** Publisher
self.pubSocket = self.zmqContext.socket(zmq.PUB)
self.pubSocket.bind(self.pub_address)
time.sleep(0.005) # mandatory sleep for ZMQ backend
self.logger.debug("[pub] Bound to {}".format(self.pub_address))
# * Initialize other members
self.valid = False
self.buttons = [0, 0] # primary, secondary
self.transform = hm.translation(hm.identity(), self.position_offset)
#self.position = self.position_offset
self.loop = True # TODO ensure this is properly shared across threads
# * Start sensing loop
self.senseThread = Thread(target=self.senseLoop)
self.senseThread.daemon = True # to prevent indefinite wait on recv()
self.senseThread.start()
time.sleep(0.005) # sleep to allow child thread to run
def set_tcp_keepalive(sock, opts=None):
# Warn if ZMQ < 3.2
try:
zmq_version_info = zmq.zmq_version_info()
except AttributeError:
# PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
# using zmq.zmq_version() and build a version info tuple.
zmq_version_info = tuple(
[int(x) for x in zmq.zmq_version().split('.')]
)
if zmq_version_info < (3, 2):
log.warning(
'You have a version of ZMQ less than ZMQ 3.2! There are '
'known connection keep-alive issues with ZMQ < 3.2 which '
'may result in loss of contact with minions. Please '
'upgrade your ZMQ!'
)
if hasattr(zmq, 'TCP_KEEPALIVE') and opts:
if 'tcp_keepalive' in opts:
sock.setsockopt(
zmq.TCP_KEEPALIVE, opts['tcp_keepalive']
)
if 'tcp_keepalive_idle' in opts:
sock.setsockopt(
zmq.TCP_KEEPALIVE_IDLE, opts['tcp_keepalive_idle']
)
if 'tcp_keepalive_cnt' in opts:
sock.setsockopt(
zmq.TCP_KEEPALIVE_CNT, opts['tcp_keepalive_cnt']
)
if 'tcp_keepalive_intvl' in opts:
sock.setsockopt(
zmq.TCP_KEEPALIVE_INTVL, opts['tcp_keepalive_intvl']
)
return sock
def test_zmq_version(self):
v = zmq.zmq_version()
self.assertTrue(isinstance(v, str))
#-----------------------------------------------------------------------------
import os
import warnings
import zmq
if os.name == 'nt':
if zmq.__version__ < '2.1.7':
raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.7 on Windows, "
"and you appear to have %s"%zmq.__version__)
elif zmq.__version__ < '2.1.4':
raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)
if zmq.zmq_version() >= '3.0.0':
warnings.warn("""libzmq 3 detected.
It is unlikely that IPython's zmq code will work properly.
Please install libzmq stable, which is 2.1.x or 2.2.x""",
RuntimeWarning)
from IPython.utils.pickleutil import Reference
from .client.asyncresult import *
from .client.client import Client
from .client.remotefunction import *
from .client.view import *
from .controller.dependency import *
except Exception, e:
logger.error(e)
finally:
logger.info("Exiting..")
if __name__ == '__main__':
# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-d', "--debug", action="store_true", help="Enable Debugging")
parser.add_argument('-c', '--conf', required=True, help="JSON Config File")
args = parser.parse_args()
# configure logging
logging.basicConfig(format="%(asctime)s [%(name)s:%(lineno)d][%(levelname)s] %(message)s", level=logging.INFO)
# enable debugging
if args.debug:
logger.setLevel(logging.DEBUG)
# open config and parse
with open(args.conf) as data_file:
conf = json.load(data_file)
# log configuration
logger.info('zmq python bindings %s, libzmq version %s' % (repr(zmq.__version__), zmq.zmq_version()))
# run
read(conf)
import sys
logger.info('Python Version: %s' % sys.version)
logger.info('Platform: %s' % sys.platform)
except Exception:
logger.error('Failed to find python version')
try:
import sys
logger.info('windows version: %s' % str(sys.getwindowsversion()))
except Exception:
pass
try:
import zmq
logger.info('PyZMQ Version: %s' % zmq.__version__)
logger.info('ZMQ Version: %s' % zmq.zmq_version())
except Exception:
logger.error('Failed to find PyZMQ version')
try:
import h5py
logger.info('h5py Version: %s' % h5py.version.info)
except Exception:
logger.error('Failed to find h5py version')
try:
if 'PySide' in sys.modules.copy():
import PySide
logger.info('PySide Version: %s' % PySide.__version__)
logger.info('Qt Version: %s' % PySide.QtCore.__version__)
else:
#!/usr/bin/python
# -*- coding: latin-1 -*-
import socket
import threading
import time
from queue import Queue
import zmq
print("Current libzmq version is %s" % zmq.zmq_version())
print("Current pyzmq version is %s" % zmq.__version__)
class Server(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self, args=(queue, ))
self.ctx = zmq.Context()
self.q = queue
def setup_listening_socket(self):
self.listener = self.ctx.socket(zmq.SUB)
self.listener.setsockopt_string(zmq.SUBSCRIBE, "HERE")
def setup_talking_socket(self):
self.talker = self.ctx.socket(zmq.ROUTER)
self.talker.bind('tcp://*:5556')
def run(self):
self.setup_listening_socket()
self.setup_talking_socket()
server.send(b"Hello")
if client.poll(1000):
msg = client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# stop auth thread
auth.stop()
if __name__ == '__main__':
if zmq.zmq_version_info() < (4, 0):
raise RuntimeError(
"Security is not supported in libzmq version < 4.0. libzmq version {0}".format(
zmq.zmq_version()
)
)
if '-v' in sys.argv:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
run()
while (True):
try:
msg = socket.recv_json(zmq.NOBLOCK)
except zmq.error.Again:
break
except Exception, e:
logging.info(e)
break
else:
logging.info(pprint.pformat(msg))
except Exception, e:
logging.info(e)
finally:
raw_input('Press Enter to continue...')
if (__name__ == '__main__'):
logging.info('zmq python bindings %s, libzmq version %s' %
(repr(zmq.__version__), zmq.zmq_version()))
parser = argparse.ArgumentParser(
description='Verbose QuakeLive server statistics')
parser.add_argument('--host',
default=HOST,
help='ZMQ URI to connect to. Defaults to %s' % HOST)
parser.add_argument('--password', required=False)
args = parser.parse_args()
#logging.debug( repr( args ) )
verbose(args)
client.curve_serverkey = server_public
client.connect('tcp://127.0.0.1:9000')
server.send(b"Hello")
if client.poll(1000):
msg = client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# stop auth thread
auth.stop()
if __name__ == '__main__':
if zmq.zmq_version_info() < (4, 0):
raise RuntimeError(
"Security is not supported in libzmq version < 4.0. libzmq version {0}"
.format(zmq.zmq_version()))
if '-v' in sys.argv:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
run()
def run(self, ip, port, register=False):
def _invalid(_):
self._send_error('Unknown command')
return False
try:
self.__load_keychain()
except Exception as err:
raise Exception('Unable to load issuer keychain')
try:
self.output('Launching CA listener')
context = zmq.Context()
self.output("Listening socket use ZMQ version {v}".format(v=zmq.zmq_version()), level="DEBUG")
self._socket = context.socket(zmq.REP)
self._socket.bind('tcp://{host}:{port}'.format(host=ip, port=port))
self.output("Listener Socket bind to tcp://{host}:{port}".format(host=ip, port=port))
except zmq.ZMQError as err:
raise upkica.core.UPKIError(20,"Stalker process failed with: {e}".format(e=err))
except Exception as err:
raise upkica.core.UPKIError(20,"Error on connection: {e}".format(e=err))
self._run = True
while self._run:
try:
msg = self._socket.recv_json()
except zmq.ZMQError as e:
self.output('ZMQ Error: {err}'.format(err=e), level="ERROR")
continue
except ValueError:
self.output('Received unparsable message', level="ERROR")
continue
except SystemExit:
self.output('Poison listener...', level="WARNING")
break
try:
self.output('Receive {task} action...'.format(task=msg['TASK']), level="INFO")
self.output('Action message: {param}'.format(param=msg), level="DEBUG")
task = "_upki_{t}".format(t=msg['TASK'].lower())
except KeyError:
self.output('Received invalid message', level="ERROR")
continue
try:
params = msg['PARAMS']
except KeyError:
params = {}
func = getattr(self, task, _invalid)
try:
res = func(params)
except Exception as err:
self.output('Error: {e}'.format(e=err), level='error')
self._send_error(err)
continue
if res is False:
continue
try:
self._send_answer(res)
except Exception as err:
self.output('Error: {e}'.format(e=err), level='error')
self._send_error(err)
continue
