class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Workers exiting..."
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg, flags=NOBLOCK)
if self._shutdown:
break
self.cleanup()
def main(name):
map = generate_map(name)
context = Context()
pub_socket = context.socket(PUB)
pub_socket.bind('tcp://0.0.0.0:20000')
map['pub_server'] = pub_socket
rep_socket = context.socket(REP)
rep_socket.bind('tcp://0.0.0.0:20001')
map['rep_server'] = rep_socket
manager = MissionManager()
rep_socket = map['rep_server']
manager.start_mission(LocMission(map, None))
return
while True:
req = MissionMessage()
req.ParseFromString(rep_socket.recv())
if req.type == 6:
req.type = 7
try:
if req.missionType == MissionMessage.CONNECTION:
manager.start_mission(PlaceMission(map, (0.0, 0.0), req))
elif req.missionType == MissionMessage.LOCALIZATION:
manager.start_mission(LocMission(map, req))
else:
raise ValueError('unknown mission type')
req.status = MissionMessage.ACTIVE
except RuntimeError:
req.status = MissionMessage.REJECTED
rep_socket.send(req.SerializeToString())
async def get(socket_id: SocketStruct,
msg: bytes,
ctx: zmq.Context,
timeout=500,
linger=2000,
retries=10,
dealer=False):
if retries < 0:
return None
if dealer:
socket = ctx.socket(zmq.DEALER)
else:
socket = ctx.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, linger)
try:
# Allow passing an existing socket to save time on initializing a _new one and waiting for connection.
socket.connect(str(socket_id))
await socket.send(msg)
event = await socket.poll(timeout=timeout, flags=zmq.POLLIN)
if event:
response = await socket.recv()
socket.close()
return response
else:
socket.close()
return None
except Exception as e:
socket.close()
return await get(socket_id, msg, ctx, timeout, linger, retries - 1)
def __init__(self, context: zmq.Context):
self.pull_socket = context.socket(zmq.PULL)
self.pull_socket.connect(self.pull_address)
self.sub_socket = context.socket(zmq.SUB)
self.sub_socket.connect(self.sub_address)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, b"10001")
class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Workers exiting..."
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg, flags=NOBLOCK)
if self._shutdown:
break
self.cleanup()
class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
def cleanup(self):
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg)
if self._shutdown:
break
self.cleanup()
def run(port):
""" Run a translations server at a specific port.
It always listens on all available network devices!
"""
context = Context(1)
sync_socket = context.socket(ROUTER)
sync_socket.bind(_SYNC_ENDPOINT)
frontend = context.socket(ROUTER)
frontend.bind("tcp://*:{}".format(port))
# Socket facing services
backend = context.socket(DEALER)
backend.bind(_REQUEST_ENDPOINT)
try:
worker_threads, worker_identities = _start_workers(
context, sync_socket, config.WORKERS, 1000)
_LOG.debug("Running device...")
try:
proxy(frontend, backend)
except KeyboardInterrupt:
print("\rShutting down...")
frontend.close()
frontend = None
_shut_down_workers(sync_socket, worker_threads, worker_identities, 5)
finally:
if frontend is not None:
frontend.close()
backend.close()
sync_socket.close()
_LOG.debug("Done")
class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
def cleanup(self):
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg)
if self._shutdown:
break
self.cleanup()
class wrapped_dispatcher(object):
def __init__(self, enqueued=None, on_load=None):
self.queue = Queue()
kwargs = {
'queue': self.queue
}
if enqueued:
kwargs['enqueued_tasks'] = enqueued
if on_load:
kwargs['on_daemon_load'] = on_load
self.dispatcher = WrappedDispatcher(**kwargs)
self.context = None
self.sockets = {}
def __enter__(self):
self.dispatcher.start()
self.context = Context()
self.sockets['in'] = self.context.socket(PUSH)
self.sockets['out'] = self.context.socket(PULL)
self.sockets['in'].connect(settings.ZTASKD_URL)
self.sockets['out'].connect(settings.ZTASK_WORKER_URL)
return (self.queue, self.sockets['in'], self.sockets['out'])
def __exit__(self, exc_type, exc_value, traceback):
self.dispatcher.terminate()
self.context.destroy()
self.queue.close()
def __init__(self, context: zmq.Context, tasks_to_send: int):
self.tasks_to_send = tasks_to_send
self.socket = context.socket(zmq.PUSH)
self.socket.bind(self.address)
self.sink_socket = context.socket(zmq.PUSH)
self.sink_socket.connect(self.sink_address)
class TestEchoService(WindmillTestCase):
def setUp(self):
self.zmq_ctx = Context()
d = 'test_out'
if not os.path.exists(d):
os.makedirs(d)
def tearDown(self):
pass
def test_echo_service_default_behavior(self):
req_out_sock = self.zmq_ctx.socket(REQ)
req_out_sock.bind('tcp://*:8889')
t = thread_wrap_windmill('EchoService')
try:
t.start()
self.assertTrue(t.is_alive,
'The EchoService instance should have started.')
req_out_sock.send("echo, echo, echo")
msg = req_out_sock.recv()
self.assertEqual("echo, echo, echo", msg)
finally:
t.windmill.kill()
t.join(3)
self.assertFalse(t.is_alive(),
'The EchoService instance should have shutdown.')
req_out_sock.close()
def test_echo_service_options(self):
req_out_sock = self.zmq_ctx.socket(REQ)
req_out_sock.bind('tcp://*:8899')
argv = ['-m', 'pong', '--reply_sock_url', 'tcp://localhost:8899']
t = thread_wrap_windmill('EchoService', argv=argv)
try:
t.start()
self.assertTrue(t.is_alive(),
'The EchoService instance should have started.')
req_out_sock.send('ping')
msg = req_out_sock.recv()
self.assertEqual('pong', msg)
finally:
t.windmill.kill()
t.join(3)
self.assertFalse(t.is_alive(),
'The EchoService instance shold have shutdown.')
req_out_sock.close()
class SubPrice:
def __init__(self, prodcode, addr=f'tcp://{server_IP}:6869'):
self._ctx = Context()
self._sub_socket = self._ctx.socket(zmq.SUB)
self._sub_socket.set_string(zmq.SUBSCRIBE, '')
self._sub_socket.setsockopt(zmq.RCVTIMEO, 5000)
self._req_price_socket = self._ctx.socket(zmq.REQ)
self._req_price_socket.connect(f'tcp://{server_IP}:6870')
self._addr = addr
self._prodcode = prodcode
self.__is_active = False
self.__is_sub = False
self.__thread = Thread()
self._spfunc = SpFunc()
def __run(self, func):
while self.__is_active:
try:
price = self._sub_socket.recv_pyobj()
func(price)
except zmq.ZMQError:
...
def __call__(self, func):
self._func = func
return self
def start(self):
if not self.__is_active:
self.__is_active = True
self._sub_socket.connect(self._addr)
self.__thread = Thread(target=self.__run, args=(self._func, ))
self.__thread.setDaemon(True)
self.__thread.start()
def stop(self):
self.__is_active = False
self.__thread.join()
self._sub_socket.disconnect(self._addr)
def is_alive(self):
return self.__is_active
def sub(self):
self._spfunc.sub_price(self._prodcode)
self.__is_sub = True
def unsub(self):
self._spfunc.unsub_price(self._prodcode)
self.__is_sub = False
def get_price(self):
self._req_price_socket.send_string(self._prodcode)
price = self._req_price_socket.recv_pyobj()
return price
def __init__(self, context: zmq.Context):
self.pull_socket = context.socket(zmq.PULL)
self.pull_socket.connect(self.pull_address)
self.sub_socket = context.socket(zmq.SUB)
self.sub_socket.connect(self.sub_address)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, b"10001")
self.poller = zmq.Poller()
self.poller.register(self.pull_socket, zmq.POLLIN)
self.poller.register(self.sub_socket, zmq.POLLIN)
class Master(object):
def __init__(self, full_socket_address):
self.context = Context()
self.workers = OrderedDict()
self.overflow_launch = False
self.stats = False
self.full_socket_address = full_socket_address
self.socket_address, self.socket_port = full_socket_address.split(':')
@property
@zeroMQError
def init_pubsocket(self):
''' initialise la socket pour permettre de lancer
le benchmark via un publish
'''
self.pubsocket = self.context.socket(PUB)
self.pubsocket.bind('tcp://{}'.format(self.full_socket_address))
@property
@zeroMQError
def init_repsocket(self):
''' init la socket pour permettre de repondre à un
nouveau worker qui vient s'ajouter dynamiquement
par default ce port est fixe (55555)
'''
self.repsocket = self.context.socket(REP)
self.repsocket.bind('tcp://{}:55555'.format(self.socket_address))
@property
def wait_workers(self):
''' permet l'ajout de worker en attendand le
message pour lancer le benchmark
'''
while not self.overflow_launch:
message = loads(self.repsocket.recv_json())
# workers
if '_id' in message:
self.workers[message['_id']] = 'ready'
self.repsocket.send('ok')
sys.stdout.write('worker {} is ready\n'.format(message['_id']))
# overflow signals
elif 'overflow' in message:
self.repsocket.send('ok')
sys.stdout.write('master: launch overflow for {}\n'.format(self.workers.keys()))
self.launch_benchmark
@property
def launch_benchmark(self):
''' declenche le benchmark
'''
self.pubsocket.send('OVERFLOW')
self.workers = OrderedDict()
class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Home exiting..."
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
class DrillingWell(Thread):
def __init__(self):
super(DrillingWell, self).__init__(name="DrillingWell")
self.context = Context()
self.push = self.context.socket(PUSH)
self.push.bind("tcp://*:7000")
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Producer exiting..."
self.push.close()
self.context.term()
def run(self):
count = 0
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
sleep(0.01)
count += 1
self.push.send("SOMETHING " + str(count))
if self._shutdown:
break
self.cleanup()
class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Home exiting..."
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
def _register(self, context: zmq.Context, identity: bytes, data: Any):
if identity in self.queues:
return
# Create a new in-process channel for this handler
channel = self.channels.pop()
interconnect = RemoteHandler.interconnect_channel(channel)
# Create the handler's zmq upstream socket to send data
queue = context.socket(zmq.PUSH)
queue.bind(interconnect)
# Create the handler thread
handler = RemoteHandler(context, channel, self.client_config, deserialize_int(data))
handler.start()
handler.ready.wait()
# Update the database
self.handlers[identity] = handler
self.queues[identity] = queue
# Put the next available channel to the queue
# When workers get de-registered they will also add themselves to this queue
# This lets us reuse channels
self.latest_channel += 1
self.channels.append(self.latest_channel)
class WorkerTest(TestCase):
"""Ensures the worker correctly handles messages
"""
def setUp(self):
self.queue = Queue()
self.context = Context()
self.socket = self.context.socket(PUSH)
self.socket.bind(settings.ZTASK_WORKER_URL)
self.worker = WrappedWorker(queue=self.queue)
self.worker.start()
def tearDown(self):
self.worker.terminate()
self.context.destroy()
def test_exec(self):
"""Tests executing a task
"""
uuid = str(uuid4())
self.socket.send_pyobj((uuid,))
self.assertEqual(
self.queue.get(),
uuid
)
self.assertTrue(self.queue.get())
self.queue.close()
def on_message(self, message):
tcp = HSD.get_tcp()
poller = zmq.Poller()
ctx1 = Context()
sub_socket = ctx1.socket(zmq.SUB)
sub_socket.connect('tcp://{}:6868'.format(tcp))
sub_socket.setsockopt_unicode(zmq.SUBSCRIBE, '')
poller.register(sub_socket, zmq.POLLIN)
while 1: # 循环推送数据
for handler in WebSocketHandler.socket_handlers:
ticker = sub_socket.recv_pyobj()
this_time = ticker.TickerTime
objArr = cache.get("objArr")
times, opens, high, low, close, vol = objArr if objArr else (
ticker.TickerTime * 1000, ticker.Price, ticker.Price, ticker.Price, ticker.Price, ticker.Qty)
GetRealTimeData(ticker.TickerTime, ticker.Price, ticker.Qty)
# print(times,opens,high,low,close)
self.zs = 0
if time.localtime(this_time).tm_min != time.localtime(times / 1000).tm_min:
tm = time.localtime(times / 1000)
tm = datetime.datetime(tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour, tm.tm_min)
self.zs = self.zbjs.send((tm, opens, high, low, close))
self.zs = self.zs[tm]['datetimes'][-1][1] if self.zs[tm]['datetimes'] else 0
if this_time * 1000 != times:
data = {'times': str(times), 'opens': str(opens), 'high': str(high), 'low': str(low),
'close': str(close), 'vol': str(vol), 'zs': str(self.zs)}
data = json.dumps(data).encode()
handler.write_message(data)
def _create_sending_dealer_socket(
context: zmq.Context, connection_identity: str,
connection_replication_port) -> zmq.Socket:
"""
Creates a socket for sending messages to another node.
Parameters
----------
context
ZMQ context to use to create the socket
Returns
-------
socket
Dealer socket on specified port
Warnings
---------
You must call this from the same thread the context was created in.
"""
dealer_socket = context.socket(zmq.DEALER)
dealer_socket.set_string(zmq.IDENTITY, connection_identity)
dealer_socket.setsockopt(zmq.LINGER, 0)
dealer_socket.connect(f"tcp://127.0.0.1:{connection_replication_port}")
return dealer_socket
class DrillingWell(Thread):
def __init__(self):
super(DrillingWell, self).__init__(name="DrillingWell")
self.context = Context()
self.push = self.context.socket(PUSH)
self.push.bind("tcp://*:7000")
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Producer exiting..."
self.push.close()
self.context.term()
def run(self):
count = 0
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
sleep(0.01)
count += 1
self.push.send("SOMETHING " + str(count))
if self._shutdown:
break
self.cleanup()
class ZeroLeakServer(object):
def __init__(self, bind_addr, storage_type, **storage_kwargs):
"""
:param str bind_addr: Bind the socket to an address.
:param str storage_type: The storage type.
:param dict storage_kwargs: list of parameters to instanciate the storage.
"""
self.bind_addr = bind_addr
self.storage_type = storage_type
self.storage_kwargs = storage_kwargs
def _init_socket(self):
self.context = Context()
self.socket = self.context.socket(SUB)
self.socket.setsockopt(SUBSCRIBE, "zeroleak")
self.socket.bind("tcp://{}".format(self.bind_addr))
def run(self):
"""
"""
storage = getattr(leakdb, self.storage_type, None)
if not storage:
raise UnknownStorageException(self.storage_type)
self.storage = storage(**self.storage_kwargs)
self._init_socket()
while True:
json_msg = self.socket.recv_json()
pass
class Zmq_broker(BaseModule):
context = None
s_pub = None
pub_endpoint = None
serialize_to = None
serialize = None
def __init__(self, mod_conf, pub_endpoint, serialize_to):
from zmq import Context, PUB
BaseModule.__init__(self, mod_conf)
self.pub_endpoint = pub_endpoint
self.serialize_to = serialize_to
logger.info("[Zmq Broker] Binding to endpoint " + self.pub_endpoint)
# This doesn't work properly in init()
# sometimes it ends up beings called several
# times and the address becomes already in use.
self.context = Context()
self.s_pub = self.context.socket(PUB)
self.s_pub.bind(self.pub_endpoint)
# Load the correct serialization function
# depending on the serialization method
# chosen in the configuration.
if self.serialize_to == "msgpack":
from msgpack import Packer
packer = Packer(default=encode_monitoring_data)
self.serialize = lambda msg: packer.pack(msg)
elif self.serialize_to == "json":
self.serialize = lambda msg: json.dumps(msg, cls=SetEncoder)
else:
raise Exception("[Zmq Broker] No valid serialization method defined (Got " + str(self.serialize_to) + ")!")
# Called by Broker to say 'let's prepare yourself guy'
def init(self):
logger.info("[Zmq Broker] Initialization of the Zmq broker module")
# Publish to the ZeroMQ socket
# using the chosen serialization method
def publish(self, msg, topic=""):
from zmq import SNDMORE
data = self.serialize(msg)
self.s_pub.send(topic, SNDMORE)
self.s_pub.send(data)
# An host check have just arrived, we UPDATE data info with this
def manage_brok(self, b):
logger.debug("[Zmq Broker] Got broker update: " + str(b.data))
# Publish update data to the ZeroMQ endpoint.
msg = b.data
self.publish(msg, b.type)
# Properly close down this thing.
def do_stop(self):
self.s_pub.close()
self.context.term()
class WorkManager(object):
def __init__(self, worker_id, outside_ros=False):
self.worker_id = worker_id
self.outside_ros = outside_ros
if self.outside_ros:
rospy.logwarn('Controller is using ZMQ to get work')
self.context = Context()
self.socket = self.context.socket(REQ)
self.socket.connect('tcp://127.0.0.1:33589')
else:
rospy.logwarn('Controller is using ROS to get work')
self.services = {'get': {'name': '/work/get', 'type': GetWork},
'update': {'name': '/work/update', 'type': UpdateWorkStatus}}
for service_name, service in self.services.items():
rospy.loginfo("Controller is waiting service {}...".format(service['name']))
rospy.wait_for_service(service['name'])
service['call'] = rospy.ServiceProxy(service['name'], service['type'])
def get(self):
if self.outside_ros:
self.socket.send_json({"type": "get", "worker": self.worker_id})
return GetWorkResponse(**self.socket.recv_json())
else:
call = self.services['get']['call']
return call(GetWorkRequest(worker=self.worker_id))
def update(self, task, trial, iteration, success):
if self.outside_ros:
self.socket.send_json({"type": "update", "task": task, "trial": trial, "iteration": iteration, "worker": self.worker_id, "success": success})
return UpdateWorkStatusResponse(**self.socket.recv_json())
else:
call = self.services['update']['call']
return call(UpdateWorkStatusRequest(task=task, trial=trial, iteration=iteration,
worker=self.worker_id, success=success))
class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
def cleanup(self):
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
def initialize_zmq_socket(host, port):
logger.info("Initializing ZMQ consumer socket: Host: %s, Port: %d", host, port)
context = Context()
zmq_socket = context.socket(PULL)
zmq_socket.connect("tcp://{0}:{1}".format(host, port))
logger.info("ZMQ consumer socker initilized.")
return zmq_socket
class Listener(Thread):
def __init__(self):
super(Listener, self).__init__(name="Listener")
self._shutdown = False
self.context = Context()
self.sub = self.context.socket(SUB)
self.sub.bind('tcp://*:7000')
self.sub.setsockopt(SUBSCRIBE, "")
self.poller = Poller()
self.poller.register(self.sub, POLLIN)
def cleanup(self):
self.sub.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.sub) == POLLIN:
msg = self.sub.recv(flags=NOBLOCK)
print msg
if self._shutdown:
break
self.cleanup()
class Listener(Thread):
def __init__(self):
super(Listener, self).__init__(name="Listener")
self._shutdown = False
self.context = Context()
self.sub = self.context.socket(SUB)
self.sub.bind('tcp://*:7000')
self.sub.setsockopt(SUBSCRIBE, "")
self.poller = Poller()
self.poller.register(self.sub, POLLIN)
def cleanup(self):
self.sub.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.sub) == POLLIN:
msg = self.sub.recv(flags=NOBLOCK)
print msg
if self._shutdown:
break
self.cleanup()
def init_connections(self):
"""
Initialise zmq sockets, poller and queues.
Because this class is s Process, This method must be call in the run
method to be hold by the correct process.
"""
# Prepare our context and sockets
context = Context()
# Socket to talk to prev_stages
for name,connections in self.connections.items():
self.queue_limit[name] = connections[2]
sock_router = context.socket(ROUTER)
try:
sock_router.bind('tcp://*:' + connections[0])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'
.format(e, connections[0]))
return False
self.router_sockets[name] = sock_router
# Socket to talk to next_stages
sock_dealer = context.socket(ROUTER)
try:
sock_dealer.bind("tcp://*:" + connections[1] )
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'
.format(e, connections[1]))
return False
self.dealer_sockets[name] = sock_dealer
self.next_available_stages[name] = list()
self.queue_jobs[name] = list()
# Use a ZMQ Pool to get multichannel message
self.poller = Poller()
# Register dealer socket to next_stage
for n, dealer in self.dealer_sockets.items():
self.poller.register(dealer, POLLIN)
for n, router in self.router_sockets.items():
self.poller.register(router, POLLIN)
# Register router socket to prev_stages or producer
self.socket_pub = context.socket(PUB)
if self.gui_address is not None:
try:
self.socket_pub.connect("tcp://" + self.gui_address)
except ZMQError as e:
self.log.error("".format(e, self.gui_address))
return False
# This flag stop this current process
return True
def init_connexions(self):
"""
Initialise zmq sockets, poller and queues.
Because this class is s Process, This method must be call in the run
method to be hold by the correct processus.
"""
# Prepare our context and sockets
context = Context()
# Socket to talk to prev_stages
for name, connexions in self.connexions.items():
self.queue_limit[name] = connexions[2]
sock_router = context.socket(ROUTER)
try:
sock_router.bind('tcp://*:' + connexions[0])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[0]))
return False
self.router_sockets[name] = sock_router
# Socket to talk to next_stages
sock_dealer = context.socket(ROUTER)
try:
sock_dealer.bind("tcp://*:" + connexions[1])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[1]))
return False
self.dealer_sockets[name] = sock_dealer
self.next_available_stages[name] = list()
self.queue_jobs[name] = list()
# Use a ZMQ Pool to get multichannel message
self.poller = Poller()
# Register dealer socket to next_stage
for n, dealer in self.dealer_sockets.items():
self.poller.register(dealer, POLLIN)
for n, router in self.router_sockets.items():
self.poller.register(router, POLLIN)
# Register router socket to prev_stages or producer
self.socket_pub = context.socket(PUB)
if self.gui_address is not None:
try:
self.socket_pub.connect("tcp://" + self.gui_address)
except ZMQError as e:
self.log.error("".format(e, self.gui_address))
return False
# This flag stop this current processus
return True
class Zmq_broker(BaseModule):
context = None
s_pub = None
pub_endpoint = None
serialize_to = None
serialize = None
def __init__(self, mod_conf, pub_endpoint, serialize_to):
from zmq import Context, PUB
BaseModule.__init__(self, mod_conf)
self.pub_endpoint = pub_endpoint
self.serialize_to = serialize_to
logger.info("[Zmq Broker] Binding to endpoint " + self.pub_endpoint)
# This doesn't work properly in init()
# sometimes it ends up beings called several
# times and the address becomes already in use.
self.context = Context()
self.s_pub = self.context.socket(PUB)
self.s_pub.bind(self.pub_endpoint)
# Load the correct serialization function
# depending on the serialization method
# chosen in the configuration.
if self.serialize_to == "msgpack":
from msgpack import Packer
packer = Packer(default=encode_monitoring_data)
self.serialize = lambda msg: packer.pack(msg)
elif self.serialize_to == "json":
self.serialize = lambda msg: json.dumps(msg, cls=SetEncoder)
else:
raise Exception(
"[Zmq Broker] No valid serialization method defined (Got " +
str(self.serialize_to) + ")!")
# Called by Broker to say 'let's prepare yourself guy'
def init(self):
logger.info("[Zmq Broker] Initialization of the Zmq broker module")
# Publish to the ZeroMQ socket
# using the chosen serialization method
def publish(self, msg, topic=""):
from zmq import SNDMORE
data = self.serialize(msg)
self.s_pub.send(topic, SNDMORE)
self.s_pub.send(data)
# An host check have just arrived, we UPDATE data info with this
def manage_brok(self, b):
logger.debug("[Zmq Broker] Got broker update: " + str(b.data))
#Publish update data to the ZeroMQ endpoint.
msg = b.data
self.publish(msg, b.type)
# Properly close down this thing.
def do_stop(self):
self.s_pub.close()
self.context.term()
def __init__(self, joystick_position_subject: JoystickPositionSubject,
ip: str, port: int, zmq_context: zmq.Context) -> None:
super().__init__()
self.joystick_position_subject = joystick_position_subject
self.joystick_position_subject.attach(self)
self.socket = zmq_context.socket(zmq.PAIR)
self.socket.bind("tcp://{}:{}".format(
ip, port)) # this class is the server
def initialize_zmq_socket(host, port):
logger.info("Initializing ZMQ consumer socket: Host: %s, Port: %d", host,
port)
context = Context()
zmq_socket = context.socket(PULL)
zmq_socket.connect("tcp://{0}:{1}".format(host, port))
logger.info("ZMQ consumer socker initilized.")
return zmq_socket
def step2(context: zmq.Context = None):
"""Step 2"""
context = context or zmq.Context.instance()
# Bind to inproc: endpoint, then start upstream thread
receiver = context.socket(zmq.PAIR)
receiver.bind("inproc://step2")
thread = threading.Thread(target=step1)
thread.start()
# Wait for signal
msg = receiver.recv()
# Signal downstream to step 3
sender = context.socket(zmq.PAIR)
sender.connect("inproc://step3")
sender.send(b"")
class ZeroLeakClient(object):
def __init__(self, connect_address):
self.connect_address = connect_address
def __init_socket(self):
self.context = Context()
self.socket = self.context.socket(PUB)
self.socket.connect("tcp://{}".format(self.connect_address))
def __init__(self, context: Context, protocol: str, ip: str, port: int,
socket_type: int):
self.protocol = protocol
self.ip = ip
self.port = port
self.context = context
self.socket: Socket = context.socket(socket_type)
self.socket.connect(f"{protocol}://{ip}:{port}")
def run(self):
context = Context()
socket = context.socket(self.socket_type)
socket.bind(self.socket_url)
data = socket.recv_pyobj()
self.queue.put(data)
context.destroy()
self.queue.close()
class Worker(object):
def __init__(self, master_full_socket_address):
self.context = Context()
self.worker_id = randint(1, 100000)
self.overflow_launch = False
self.master_full_socket_address = master_full_socket_address
self.master_socket_address, self.master_socket_port = master_full_socket_address.split(':')
@property
@zeroMQError
def init_subsocket(self):
''' permet de recevoir le message pour lancher le benchmark
'''
self.subsocket = self.context.socket(SUB)
self.subsocket.connect('tcp://{}'.format(self.master_full_socket_address))
self.subsocket.setsockopt(SUBSCRIBE, '')
@property
@zeroMQError
def init_reqsocket(self):
''' permet de prevenir le master de l'ajout d'un nouveau
worker
'''
self.reqsocket = self.context.socket(REQ)
self.reqsocket.connect('tcp://{}:55555'.format(self.master_socket_address))
@property
def iam_ready(self):
''' contact le master
'''
ready = False
while not ready:
msg = {'_id': self.worker_id}
self.reqsocket.send_json(dumps(msg))
if self.reqsocket.recv() == 'ok':
ready = True
sys.stdout.write('{} ready\n'.format(self.worker_id))
@property
def waiting_benchmark(self):
''' attend le message d'overflow
'''
while not self.overflow_launch:
self.subsocket.recv()
self.overflow_launch = True
def __init__(self, can_state_model: CANStateModel,
config_model: ConfigModel, context: zmq.Context):
self.can_state_model = can_state_model
self.subscriber_socket = context.socket(zmq.SUB)
self.config_model = config_model
self.conf_sub(
self.subscriber_socket, b"",
f"tcp://{self.config_model.address}:{self.config_model.can_port}")
def step1(context: zmq.Context = None):
"""Step 1"""
context = context or zmq.Context.instance()
# Signal downstream to step 2
sender = context.socket(zmq.PAIR)
sender.connect("inproc://step2")
sender.send(b"")
def __init__(self, context: zmq.Context, zipcode):
self.socket = context.socket(zmq.SUB)
print("Collecting updates from weather server…")
self.socket.connect(self.address)
self.socket.setsockopt_string(zmq.SUBSCRIBE, zipcode)
self.zipcode = zipcode
self.total_temperature = 0
class Client(object):
def __init__(self):
self._ctx = Context()
self._socket = self._ctx.socket(DEALER)
self._server_port = None
self._actions = {
'actions': self._list_actions,
'exit': self._exit,
'register': self._register,
}
def _list_actions(self):
print('''
Available actions:
actions : list available actions,
register : register client to server,
exit : terminate client
''')
def _exit(self):
exit()
def _send_msg(self, message):
pickled_message = dumps(message)
self._socket.send(pickled_message)
def _recv_msg(self):
pickled_message = self._socket.recv()
received_message = loads(pickled_message)
return received_message
def _register(self):
pseudonym = input('Enter pseudonym --> ')
cert = input('Enter certificate --> ')
message = {'type': 'register', 'pseudonym': pseudonym, 'cert': cert}
self._send_msg(message)
response = self._recv_msg()
print(response)
def execute(self):
self._server_port = input('Enter server port --> ')
self._socket.connect('tcp://localhost:%s' % self._server_port)
print('For available actions use keyword actions.')
while True:
i = input('Define action --> ')
if i in self._actions:
self._actions[i]()
else:
print('Undefined action!')
def _request_translation(self, language, country, key, plural):
""" Start up a worker, sync it and then send it a translation request.
Returns the result, shuts down the worker at the end as well.
Fails the current test, if something goes wrong.
"""
request = [
language, country if country is not None else "", key,
str(plural) if plural is not None else ""]
request = [x.encode(_ENCODING) for x in request]
context = Context()
# Create synchronization and backend sockets.
try:
sync_socket = context.socket(ROUTER)
try:
sync_socket.bind(_SYNC_ENDPOINT)
backend = context.socket(DEALER)
try:
backend.bind(_REQUEST_ENDPOINT)
worker_threads, worker_identities = _start_workers(
context, sync_socket, 1, _TIMEOUT)
poller = Poller()
poller.register(backend, POLLIN)
poller.register(sync_socket, POLLIN)
# Send request.
backend.send_multipart(
[worker_identities[0], b""] + request)
sockets = dict(poller.poll(_TIMEOUT))
# Shutdown worker.
_shut_down_workers(
sync_socket, worker_threads, worker_identities,
_TIMEOUT / 1000.0)
if backend in sockets:
# Return translation.
return backend.recv_multipart()[2].decode("utf-8")
self.fail("Worker did not response the request in time.")
finally:
backend.set(LINGER, 0)
backend.close()
finally:
sync_socket.set(LINGER, 0)
sync_socket.close()
finally:
context.destroy(linger=0)
def worker_routine(self, worker_url, context: zmq.Context = None):
context = context or zmq.Context.instance()
socket = context.socket(zmq.REP)
socket.connect(worker_url)
while True:
message = socket.recv_pyobj()
socket.send_string("ok")
self.qu.put(message)
def test_onload(self):
"""Tests onload calls
"""
context = Context()
socket = context.socket(PULL)
socket.bind('tcp://127.0.0.1:5560')
with wrapped_dispatcher(on_load=['django_ztaskq.tests.dummy_onload']) \
as __:
self.assertTrue(socket.recv_pyobj())
def initialize_zmq_socket(interface, port):
logger.info("Initializing ZMQ producer socket: Host: %s, Port: %d", interface, port)
context = Context()
zmq_socket = context.socket(PUSH)
zmq_socket.bind("tcp://{0}:{1}".format(interface, port))
logger.info("ZMQ producer socker initilized.")
return zmq_socket
def Main():
context = Context()
print("Connecting to hello world server...")
socket = context.socket(REQ)
socket.connect("udp://localhost:15555")
for i in range(10):
socket.send(b"Hello")
message = socket.recv()
print("Received reply %s [ %s ]" % (i, message))
def broadcast_socket(ctx: zmq.Context, broadcastaddr: str) -> zmq.Socket:
""" Socket for receiving broadcast message notifications
"""
LOGGER.debug("Enabling broadcast notification")
ctx = zmq.Context.instance()
sub = ctx.socket(zmq.SUB)
sub.setsockopt(zmq.LINGER, 500) # Needed for socket no to wait on close
sub.setsockopt(zmq.SUBSCRIBE, b'RESTART')
sub.connect(broadcastaddr)
return sub
class Zero(object):
def __init__(self):
self.context = Context()
@property
def subscriber(self):
try:
self._subscriber = self.context.socket(SUB)
self._subscriber.bind(ZERO_BIND_ADDRESS)
self._subscriber.setsockopt(SUBSCRIBE, "")
except ZMQError as error:
Logger.log('E200: {}'.format(error), 'CRITICAL')
exit(200)
except Exception as error:
Logger.log('E201: {}'.format(error), 'CRITICAL')
exit(201)
else:
Logger.log('bind subscriber on {}'.format(ZERO_BIND_ADDRESS), 'DEBUG')
return self._subscriber
@property
def publisher(self):
try:
self._publisher = self.context.socket(PUB)
self._publisher.connect(ZERO_CONNECT_ADDRESS)
except ZMQError as error:
Logger.log('E202: {}'.format(error), 'CRITICAL')
exit(202)
except Exception as error:
Logger.log('E203: {}'.format(error), 'CRITICAL')
exit(203)
else:
Logger.log('connection publisher on {}'.format(ZERO_CONNECT_ADDRESS), 'DEBUG')
return self._publisher
@property
def destroy(self):
self._subscriber.close()
self._publisher.close()
self.context.destroy()
Logger.log('destroy zmq socket', 'DEBUG')
class Socket(object):
def __init__(self, addr, stype):
self._context = Context()
self._socket = self._context.socket(stype)
if stype in [REP, PUB]:
self._socket.bind(addr)
else:
self._socket.connect(addr)
def __del__(self, *args, **kwargs):
self._socket.close()
def inicializar_puertos(self):
from zmq import Context, SUB, SUBSCRIBE
if self.modo == "cinematico":
context = Context()
self.socket_referencias = context.socket(SUB)
self.socket_referencias.connect("tcp://localhost:" + self.puerto_referencias)
self.socket_referencias.setsockopt(SUBSCRIBE, b'')
if self.modo == "dinamico":
raise NotImplementedError
if self.modo == "matematico":
raise NotImplementedError
def forward_request(self, address, message):
"""Forward a request to another server.
"""
# FIXME: shouldn't we have an open socket at all times ?
if address not in self.mirrors:
context = Context()
socket = context.socket(REQ)
socket.connect(address)
self.mirrors[address] = socket
else:
socket = self.mirrors[address]
socket.send(str(message))
return socket.recv()
def forward_request(self, address, message):
"""Forward a request to another server.
"""
if address not in self.mirrors:
context = Context()
socket = context.socket(REQ)
socket.setsockopt(LINGER, 1)
socket.connect(address)
self.mirrors[address] = socket
else:
socket = self.mirrors[address]
socket.send(str(message))
return socket.recv()
def __init__(self):
# pygame
self.screen = pygame.display.set_mode((self.WIDTH, self.HEIGHT),
HWSURFACE | DOUBLEBUF, 32)
pygame.display.set_caption('kbsim - 0.0s')
self.clock = pygame.time.Clock()
self.world = world(gravity=(0, 0), doSleep=True)
# zqm
context = Context()
self.socket = context.socket(PAIR)
self.socket.connect('tcp://localhost:{}'.format(self.ZMQ_PORT))
class zmq_listen_connection(object):
'''zmq监听连接'''
def __init__(self,port_number,callback_function):
'''
端口号
指定处理消息回调函数，消息作为入参
'''
super(zmq_listen_connection, self).__init__()
self.__context=Context()
self.__socket=self.__context.socket(PULL)
self.__socket.bind("tcp://0.0.0.0:%d"%(port_number))
self.__stream_pull=ZMQStream(self.__socket)
self.__stream_pull.on_recv(callback_function)
pass
pass
def __init__(self):
# pygame
self.screen = pygame.display.set_mode((self.WIDTH, self.HEIGHT),
HWSURFACE | DOUBLEBUF, 32)
pygame.display.set_caption('kbsim')
self.clock = pygame.time.Clock()
# pybox2d
self.world = world(gravity=(0, 0), doSleep=True)
self.maze = Labyrinth(self.world, self.SCALE_REAL_TO_SIM, self.SCALE_REAL_TO_VIS)
# zqm
context = Context()
self.socket = context.socket(PAIR)
self.socket.connect('tcp://localhost:{}'.format(self.ZMQ_PORT))
def simulador(puerto_zmq, f, x0, dt):
from scipy.integrate import ode
from zmq import Context, PUB
from msgpack import packb
from matplotlib.pyplot import figure
from time import time, sleep
from numpy import sin, pi, degrees, array
context = Context()
socket = context.socket(PUB)
socket.bind("tcp://*:" + puerto_zmq)
def mandar_mensaje(señales):
socket.send(packb(señales))
fig = figure(figsize=(6,3))
ax = fig.gca()
t0 = time()
ts = [0]
ys = [array(x0)]
sis = ode(f)
sis.set_initial_value(x0, t0)
ngdl = int(len(x0)/2)
while True:
try:
ys.append(degrees(sis.integrate(sis.t + dt)))
while time() - t0 - ts[-1] < dt - 0.0004:
sleep(dt*0.01)
ts.append(time() - t0)
mandar_mensaje(ys[-1].tolist()[0:ngdl])
ax.clear()
if len(ys) > 100:
ax.plot(ts[-100:], ys[-100:])
#ax.text(0.05, 0.1,ts[-1]-ts[-2], transform=ax.transAxes)
else:
ax.plot(ts, ys)
#ax.text(0.05, 0.1,ts[-1]-ts[-2], transform=ax.transAxes)
fig.canvas.draw()
except KeyboardInterrupt:
break
return ts, ys
class ZmqSubscriber(HiddenSubscriber):
""" Subscriber class subscribing to a certain topic
Attributes:
context (zmq.Context):
socket (Socket): Socket object of ZMQ context
topic (String): Topic subscriber subscribes to
"""
def __init__(self, url, topic):
""" Initializes object
Args:
url (String): url to publish messages to
topic (String): Topic to publish messages under
"""
super(ZmqSubscriber, self).__init__(url)
self._context = Context()
self._socket = self._context.socket(SUB)
self._socket.setsockopt(SUBSCRIBE, topic)
self._socket.setsockopt(RCVTIMEO, 500) # Wait 500ms for message to arrive
self._socket.connect(url)
self._logger = logging.getLogger('ZeromqSubscriber')
def receive(self):
""" Receives a message
Returns:
String
"""
topic, message = self._socket.recv_multipart()
return message
def __enter__(self):
""" Statement used for the `` with ... as ...:`` returns
the object to use in the ``with`` block
Returns:
ZmqSubscriber
"""
return self
def __exit__(self, exc_type, exc_value, exc_tb):
""" Executed when leaving ``with`` block, regardless whether
because of an exception or normal program flow
"""
self._socket.close()
self._context.term()
