def router_dealer(client_uri, internal_uri): pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) pd.bind_in(client_uri) pd.bind_out(internal_uri) pd.setsockopt_in(zmq.IDENTITY, 'ROUTER') pd.setsockopt_out(zmq.IDENTITY, 'DEALER') return pd
def __setupQueueProcess(self): ''' Set up a zmq Queue ProcessDevice ''' self.processDevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) self.processDevice.bind_in("%s://%s:%s" % (CONST.FRONTEND_SOCKET["proto"],CONST.FRONTEND_SOCKET["addr"],CONST.FRONTEND_SOCKET["port"])) self.processDevice.bind_out("%s://%s:%s" % (CONST.SERVICE_SOCKET["proto"],CONST.SERVICE_SOCKET["addr"],CONST.SERVICE_SOCKET["port"])) self.processDevice.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.processDevice.setsockopt_out(zmq.IDENTITY, 'DEALER')
def subscriber_manager(context): global subscriber global publisher global subscriber_topics session = Session() # Set up forwarder device logger.debug("Configuring forwarder device") forwarder_subscriber_address = session.query(RoutingDefiniton.route) \ .filter_by(name="internal_pub") \ .scalar() forwarder_publisher_address = session.query(RoutingDefiniton.route) \ .filter_by(name="internal_sub") \ .scalar() forwarder = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB) forwarder.connect_in(forwarder_subscriber_address) forwarder.setsockopt_in(zmq.SUBSCRIBE, "") logger.debug("forwarder connect in: {0}".format(forwarder_subscriber_address)) forwarder.bind_out(forwarder_publisher_address) logger.debug("forwarder bind out: {0}".format(forwarder_publisher_address)) # Set up subscriber listening to Farm-Monitor server subscriber_address = session.query(RoutingDefiniton.route)\ .filter_by(name="subscriber_address").scalar() subscriber_port = session.query(RoutingDefiniton.route)\ .filter_by(name="subscriber_port").scalar() subscriber_address = "tcp://" + subscriber_address + ":" + subscriber_port device_id = session.query(HardwareDefinition.serial_number).scalar() subscriber_topics.append(device_id) subscriber = Subscriber(context, subscriber_address, subscriber_topics) subscriber.subscriber.on_recv(callback=server_subscriber_recv) logger.debug("Subscriber listening to: {0} topics: {1}".format(subscriber_address, subscriber_topics)) # Set up publisher to forward messages from Farm-Monitor to internal publisher_address = session.query(RoutingDefiniton.route)\ .filter_by(name="internal_pub").scalar() publisher = Publisher(context, publisher_address) logger.debug("Publisher configured to: {0}".format(publisher_address)) session.close() try: logger.info("starting forwarder, Subscriber, and Publisher") # Start the forwarder forwarder.start() IOLoop.instance().start() except KeyboardInterrupt: logger.info("stopping forwarder, Subscriber, and Publisher") IOLoop.instance().stop() subscriber.shutdown() publisher.shutdown() return
def __init__(self, db, client_uri, internal_uri, lock_uri): self.client_uri = client_uri self.internal_uri = internal_uri self.rep_uri = internal_uri.replace("*", "localhost") self.lock_uri = lock_uri self.db = db self.pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.pd.bind_in(self.client_uri) self.pd.bind_out(self.internal_uri) self.pd.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.pd.setsockopt_out(zmq.IDENTITY, 'DEALER') self.running = False
def start_device(self): queuedevice = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH) queuedevice.bind_in(QUEUE_IN) queuedevice.bind_out(QUEUE_OUT) #queuedevice.setsockopt_in(zmq.HWM, 1) #queuedevice.setsockopt_out(zmq.HWM, 1) queuedevice.start()
class EntityAPIServer(multiprocessing.Process): ''' EntityAPIServer implementation as a Queue with multiple backend services''' def __init__(self): super(EntityAPIServer,self).__init__() self.processDevice = None self.workers = [] pass def setup(self): ''' Setup the necessary infrastructure for the server ''' self.__setupQueueProcess() self.__setupWorkers() def __setupQueueProcess(self): ''' Set up a zmq Queue ProcessDevice ''' self.processDevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) self.processDevice.bind_in("%s://%s:%s" % (CONST.FRONTEND_SOCKET["proto"],CONST.FRONTEND_SOCKET["addr"],CONST.FRONTEND_SOCKET["port"])) self.processDevice.bind_out("%s://%s:%s" % (CONST.SERVICE_SOCKET["proto"],CONST.SERVICE_SOCKET["addr"],CONST.SERVICE_SOCKET["port"])) self.processDevice.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.processDevice.setsockopt_out(zmq.IDENTITY, 'DEALER') def __setupWorkers(self): ''' Setup N workers depending on the application configuration ''' for i in range(CONST.WORKER_SERVICES_COUNT): print("Creating worker process:%d"% (i+1)) workerProc = EntityAPIWorker (host= CONST.SERVICE_SOCKET["addr"], port = CONST.SERVICE_SOCKET["port"] , workerID = i+1 ) self.workers.append(workerProc) def run(self): self.setup() print("Initiating Queue") self.processDevice.start() print("Starting Workers") for worker in self.workers: worker.start() for worker in self.workers: worker.join() def stop(self): if self.workers: for worker in self.workers: worker.stop()
def __init__(self, context, endpoint, procs=2, mapreduce=False): """Collect frames from video stream. Args: context (zmq.Context): Zmq context of calling thread. endpoint (str): Descriptor of collection endpoint. procs (int, optional): Number of processes devoted to decoding. Defaults to 2. mapreduce (bool, optional): Enable Map-Reduce streaming pattern. Defaults to False. """ seed = uuid.uuid1().hex self.mapreduce = mapreduce self.startedonce = False if mapreduce: # https://het.as.utexas.edu/HET/Software/pyZMQ/api/zmq.devices.html#zmq.devices.ProcessDevice self.device = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) self.device.setsockopt_in(zmq.RCVHWM, self.rcvhwm) self.device.setsockopt_out(zmq.SNDHWM, self.outhwm) self.device.bind_in(endpoint) self.device.bind_out("ipc:///tmp/decin" + seed) else: self.device = Subscriber(endpoint, seed=seed) self.decoder = Decoder(context, procs=procs, seed=seed, rcvmeta=mapreduce) self.started = False
def __init__(self, zmq_in=None, zmq_out=None): ''' Instanciate the object In : zmq_in : zmq address:port combo for input (SUB) zmq_out : zmq address:port combo for output (PUB) ''' if (zmq_in == None): self.zmq_bridge_in = 'tcp://127.0.0.1:42569' else: self.zmq_bridge_in = zmq_in if (zmq_out == None): self.zmq_bridge_out = 'tcp://127.0.0.1:42568' else: self.zmq_bridge_out = zmq_out self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB) self._processd.bind_in(self.zmq_bridge_in) self._processd.bind_out(self.zmq_bridge_out) self._processd.setsockopt_in(zmq.SUBSCRIBE, b'') self._ident = 254 self._ident_number = 1
class Plug(object): ''' Plug class to bridge the signal. Create an instance zmq.ProcessDevice SUB -> QUEUE -> PUB SUB socket has no filter ''' def __init__(self, zmq_in=None, zmq_out=None): ''' Instanciate the object In : zmq_in : zmq address:port combo for input (SUB) zmq_out : zmq address:port combo for output (PUB) ''' if (zmq_in == None): self.zmq_bridge_in = 'tcp://127.0.0.1:42569' else: self.zmq_bridge_in = zmq_in if (zmq_out == None): self.zmq_bridge_out = 'tcp://127.0.0.1:42568' else: self.zmq_bridge_out = zmq_out self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB) self._processd.bind_in(self.zmq_bridge_in) self._processd.bind_out(self.zmq_bridge_out) self._processd.setsockopt_in(zmq.SUBSCRIBE, b'') self._ident = 254 self._ident_number = 1 def plug_info(self, increment=True): if (increment): self._ident_number = self._ident_number + 1 return (self.zmq_bridge_in, self.zmq_bridge_out, self._ident_number - 1) def start(self): self._processd.start() @staticmethod def server_forever(): while (True): try: sleep(1) except (KeyboardInterrupt, SystemExit): break
class Plug(object): ''' Plug class to bridge the signal. Create an instance zmq.ProcessDevice SUB -> QUEUE -> PUB SUB socket has no filter ''' def __init__(self, zmq_in = None, zmq_out = None): ''' Instanciate the object In : zmq_in : zmq address:port combo for input (SUB) zmq_out : zmq address:port combo for output (PUB) ''' if(zmq_in == None): self.zmq_bridge_in = 'tcp://127.0.0.1:42569' else: self.zmq_bridge_in = zmq_in if(zmq_out == None): self.zmq_bridge_out = 'tcp://127.0.0.1:42568' else: self.zmq_bridge_out = zmq_out self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB) self._processd.bind_in(self.zmq_bridge_in) self._processd.bind_out(self.zmq_bridge_out) self._processd.setsockopt_in(zmq.SUBSCRIBE,b'') self._ident = 254 self._ident_number = 1 def plug_info(self, increment = True): if(increment): self._ident_number = self._ident_number + 1 return (self.zmq_bridge_in, self.zmq_bridge_out, self._ident_number - 1) def start(self): self._processd.start() @staticmethod def server_forever(): while(True): try: sleep(1) except(KeyboardInterrupt, SystemExit): break
def __init__(self, zmq_in = None, zmq_out = None): ''' Instanciate the object In : zmq_in : zmq address:port combo for input (SUB) zmq_out : zmq address:port combo for output (PUB) ''' if(zmq_in == None): self.zmq_bridge_in = 'tcp://127.0.0.1:42569' else: self.zmq_bridge_in = zmq_in if(zmq_out == None): self.zmq_bridge_out = 'tcp://127.0.0.1:42568' else: self.zmq_bridge_out = zmq_out self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB) self._processd.bind_in(self.zmq_bridge_in) self._processd.bind_out(self.zmq_bridge_out) self._processd.setsockopt_in(zmq.SUBSCRIBE,b'') self._ident = 254 self._ident_number = 1
def __init__(self, args): self.args = args self.context = zmq.Context() self.sock = self.context.socket(zmq.REP) self.pool = set(["Opp2Bot", "OppBotBattle", "OppBotBattleNoRaven", \ "OppBotTank", "OppBotBanshee", "OppBotFlashNew", "OppBotFlashNewRaven", "OppBotFlashFast", "OppBotMagic", "1", "2", "3", "4"]) self.win = dict() for p in self.pool: self.win[p] = [] if platform.system() == 'Windows': # Windows에서는 Process기반 queue device 실행 # 다른 프로토콜 사용 불가능 from zmq.devices import ProcessDevice device = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) device.bind_in(f"tcp://*:{args.frontend_port}") device.bind_out(f"tcp://*:{args.backend_port}") device.setsockopt_in(zmq.IDENTITY, 'ROUTER'.encode()) device.setsockopt_out(zmq.IDENTITY, 'DEALER'.encode()) device.start() self.sock.connect("tcp://localhost:{}".format(args.backend_port)) else: # Linux에서는 backend protocol을 tcp보다 빠른 # inproc (inter-process)를 사용하고, # thread 기반 queue device를 실행 import threading _device = threading.Thread( target=device_func, args=(self.context, f"tcp://*:{args.frontend_port}", f"inproc://backend"), ) _device.start() self.sock.connect(f"inproc://backend")
def run_zmq_server(): streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) zmq_in = f"ipc:///tmp/pd_in_{int(time())}_{randint(0, 1000)}" zmq_out = f"ipc:///tmp/pd_in_{int(time())}_{randint(0, 1000)}" streamerdevice.bind_in(zmq_in) streamerdevice.bind_out(zmq_out) streamerdevice.setsockopt_in(zmq.IDENTITY, b'PULL') streamerdevice.setsockopt_out(zmq.IDENTITY, b'PUSH') streamerdevice.start() return streamerdevice, zmq_in, zmq_out
# -*- coding: utf-8 -*- import zmq from zmq.devices import ProcessDevice pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) pd.bind_in('tcp://*:12345') pd.connect_out('tcp://127.0.0.1:5000') pd.setsockopt_in(zmq.IDENTITY, 'ROUTER') pd.setsockopt_out(zmq.IDENTITY, 'DEALER') pd.start() pd.join()
def __init__(self, args): self.args = args self.context = zmq.Context() self.sock = self.context.socket(zmq.REP) if platform.system() == 'Windows': # Windows에서는 Process기반 queue device 실행 # 다른 프로토콜 사용 불가능 from zmq.devices import ProcessDevice device = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) device.bind_in(f"tcp://*:{args.frontend_port}") device.bind_out(f"tcp://*:{args.backend_port}") device.setsockopt_in(zmq.IDENTITY, 'ROUTER'.encode()) device.setsockopt_out(zmq.IDENTITY, 'DEALER'.encode()) device.start() self.sock.connect("tcp://localhost:{}".format(args.backend_port)) else: # Linux에서는 backend protocol을 tcp보다 빠른 # inproc (inter-process)를 사용하고, # thread 기반 queue device를 실행 import threading _device = threading.Thread( target=device_func, args=(self.context, f"tcp://*:{args.frontend_port}", f"inproc://backend"), ) _device.start() self.sock.connect(f"inproc://backend")
def forwarder_device(port_in, port_out): from zmq.devices import ProcessDevice pd = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB) pd.bind_in('tcp://*:%s' % port_in) pd.bind_out('tcp://*:%s' % port_out) pd.setsockopt_in(zmq.IDENTITY, 'SUB') pd.setsockopt_in(zmq.SUBSCRIBE, "") pd.setsockopt_out(zmq.IDENTITY, 'PUB') pd.start()
def streamer_device(port_in, port_out): from zmq.devices import ProcessDevice pd = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH) pd.bind_in('tcp://*:%s' % port_in) pd.bind_out('tcp://*:%s' % port_out) pd.setsockopt_in(zmq.IDENTITY, 'PULL') pd.setsockopt_out(zmq.IDENTITY, 'PUSH') pd.start()
class Server(object): def __init__(self, db, client_uri, internal_uri, lock_uri): self.client_uri = client_uri self.internal_uri = internal_uri self.rep_uri = internal_uri.replace("*", "localhost") self.lock_uri = lock_uri self.db = db self.pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.pd.bind_in(self.client_uri) self.pd.bind_out(self.internal_uri) self.pd.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.pd.setsockopt_out(zmq.IDENTITY, 'DEALER') self.running = False def start(self): self.pd.start() self.context = zmq.Context() self.socket = self.context.socket(zmq.REP) self.socket.connect(self.rep_uri) def run(self): self.running = True while self.running: message = self.socket.recv() logging.debug("Received request: %s" % message) try: message = json.loads(message) if message["mode"] == "exec": # Make extra checks here old_stdout = sys.stdout stdout = sys.stdout = StringIO() try: co = compile(message["command"], "<remote>", "single") exec co in globals() except: output = sys.exc_info() else: output = stdout.getvalue() sys.stdout = old_stdout elif message["mode"] == "readall": output = db elif message["mode"] == "lock": output = {"locked": True, "uri": self.lock_uri} elif message["mode"] == "unlock": output = {"locked": False} else: entry = self.db for key in message["index"]: entry = entry[key] if not message.get("func"): output = entry else: func = getattr(entry, message["func"]) message["args"] = deepwrap(message["args"], entry.callbacks, entry.undocallbacks, self.db.wrapfile, skiproot=True) message["kwargs"] = deepwrap(message["kwargs"], entry.callbacks, entry.undocallbacks, self.db.wrapfile, skiproot=True) output = func(*message["args"], **message["kwargs"]) except: output = traceback.format_exc() logging.error(traceback.print_exc()) if type(output).__name__ in ['listiterator', 'dictionary-keyiterator']: output = list(output) try: output = json.dumps(output, cls=Encoder) except: output = str(output) self.socket.send(output) if message["mode"] == "lock": self.normal_socket = self.socket self.socket = zmq.Context().socket(zmq.REP) self.socket.bind(self.lock_uri.replace("localhost", "*")) logging.debug("Locked and listening on %s" % self.lock_uri) elif message["mode"] == "unlock": self.socket.close() self.socket = self.normal_socket logging.debug("Unlocked")
context = zmq.Context() client_socket = context.socket(zmq.REQ) client_socket.connect("tcp://%s:%s" % (config['client_ip'], config['client_port'])) def default(self, line): line = str(line) self.do_send(line) def do_send(self, line): self.client_socket.send_string(line) msg = self.client_socket.recv_string() print msg def do_receive(self, line): """ Should be able to remove this. """ self.client_socket.recv_string() def do_debug(self, line): ipdb.set_trace() if __name__ == '__main__': queue_process = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) queue_process.connect_out("tcp://%s:%s" % (config['server_ip'], config['server_port'])) queue_process.bind_in("tcp://%s:%s" % (config['client_ip'], config['client_port'])) queue_process.start() CLI().cmdloop(intro='Welcome to the shell, mortal.')