def router_dealer(client_uri, internal_uri):
pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
pd.bind_in(client_uri)
pd.bind_out(internal_uri)
pd.setsockopt_in(zmq.IDENTITY, 'ROUTER')
pd.setsockopt_out(zmq.IDENTITY, 'DEALER')
return pd
def __setupQueueProcess(self):
''' Set up a zmq Queue ProcessDevice '''
self.processDevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
self.processDevice.bind_in("%s://%s:%s" % (CONST.FRONTEND_SOCKET["proto"],CONST.FRONTEND_SOCKET["addr"],CONST.FRONTEND_SOCKET["port"]))
self.processDevice.bind_out("%s://%s:%s" % (CONST.SERVICE_SOCKET["proto"],CONST.SERVICE_SOCKET["addr"],CONST.SERVICE_SOCKET["port"]))
self.processDevice.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.processDevice.setsockopt_out(zmq.IDENTITY, 'DEALER')
def subscriber_manager(context):
global subscriber
global publisher
global subscriber_topics
session = Session()
# Set up forwarder device
logger.debug("Configuring forwarder device")
forwarder_subscriber_address = session.query(RoutingDefiniton.route) \
.filter_by(name="internal_pub") \
.scalar()
forwarder_publisher_address = session.query(RoutingDefiniton.route) \
.filter_by(name="internal_sub") \
.scalar()
forwarder = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
forwarder.connect_in(forwarder_subscriber_address)
forwarder.setsockopt_in(zmq.SUBSCRIBE, "")
logger.debug("forwarder connect in: {0}".format(forwarder_subscriber_address))
forwarder.bind_out(forwarder_publisher_address)
logger.debug("forwarder bind out: {0}".format(forwarder_publisher_address))
# Set up subscriber listening to Farm-Monitor server
subscriber_address = session.query(RoutingDefiniton.route)\
.filter_by(name="subscriber_address").scalar()
subscriber_port = session.query(RoutingDefiniton.route)\
.filter_by(name="subscriber_port").scalar()
subscriber_address = "tcp://" + subscriber_address + ":" + subscriber_port
device_id = session.query(HardwareDefinition.serial_number).scalar()
subscriber_topics.append(device_id)
subscriber = Subscriber(context, subscriber_address, subscriber_topics)
subscriber.subscriber.on_recv(callback=server_subscriber_recv)
logger.debug("Subscriber listening to: {0} topics: {1}".format(subscriber_address,
subscriber_topics))
# Set up publisher to forward messages from Farm-Monitor to internal
publisher_address = session.query(RoutingDefiniton.route)\
.filter_by(name="internal_pub").scalar()
publisher = Publisher(context, publisher_address)
logger.debug("Publisher configured to: {0}".format(publisher_address))
session.close()
try:
logger.info("starting forwarder, Subscriber, and Publisher")
# Start the forwarder
forwarder.start()
IOLoop.instance().start()
except KeyboardInterrupt:
logger.info("stopping forwarder, Subscriber, and Publisher")
IOLoop.instance().stop()
subscriber.shutdown()
publisher.shutdown()
return
def __init__(self, db, client_uri, internal_uri, lock_uri):
self.client_uri = client_uri
self.internal_uri = internal_uri
self.rep_uri = internal_uri.replace("*", "localhost")
self.lock_uri = lock_uri
self.db = db
self.pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.pd.bind_in(self.client_uri)
self.pd.bind_out(self.internal_uri)
self.pd.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.pd.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.running = False
def start_device(self):
queuedevice = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH)
queuedevice.bind_in(QUEUE_IN)
queuedevice.bind_out(QUEUE_OUT)
#queuedevice.setsockopt_in(zmq.HWM, 1)
#queuedevice.setsockopt_out(zmq.HWM, 1)
queuedevice.start()
class EntityAPIServer(multiprocessing.Process):
''' EntityAPIServer implementation as a Queue with multiple backend services'''
def __init__(self):
super(EntityAPIServer,self).__init__()
self.processDevice = None
self.workers = []
pass
def setup(self):
''' Setup the necessary infrastructure for the server '''
self.__setupQueueProcess()
self.__setupWorkers()
def __setupQueueProcess(self):
''' Set up a zmq Queue ProcessDevice '''
self.processDevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
self.processDevice.bind_in("%s://%s:%s" % (CONST.FRONTEND_SOCKET["proto"],CONST.FRONTEND_SOCKET["addr"],CONST.FRONTEND_SOCKET["port"]))
self.processDevice.bind_out("%s://%s:%s" % (CONST.SERVICE_SOCKET["proto"],CONST.SERVICE_SOCKET["addr"],CONST.SERVICE_SOCKET["port"]))
self.processDevice.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.processDevice.setsockopt_out(zmq.IDENTITY, 'DEALER')
def __setupWorkers(self):
''' Setup N workers depending on the application configuration '''
for i in range(CONST.WORKER_SERVICES_COUNT):
print("Creating worker process:%d"% (i+1))
workerProc = EntityAPIWorker (host= CONST.SERVICE_SOCKET["addr"], port = CONST.SERVICE_SOCKET["port"] , workerID = i+1 )
self.workers.append(workerProc)
def run(self):
self.setup()
print("Initiating Queue")
self.processDevice.start()
print("Starting Workers")
for worker in self.workers:
worker.start()
for worker in self.workers:
worker.join()
def stop(self):
if self.workers:
for worker in self.workers:
worker.stop()
def __init__(self, context, endpoint, procs=2, mapreduce=False):
"""Collect frames from video stream.
Args:
context (zmq.Context): Zmq context of calling thread.
endpoint (str): Descriptor of collection endpoint.
procs (int, optional): Number of processes devoted to decoding. Defaults to 2.
mapreduce (bool, optional): Enable Map-Reduce streaming pattern. Defaults to False.
"""
seed = uuid.uuid1().hex
self.mapreduce = mapreduce
self.startedonce = False
if mapreduce:
# https://het.as.utexas.edu/HET/Software/pyZMQ/api/zmq.devices.html#zmq.devices.ProcessDevice
self.device = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
self.device.setsockopt_in(zmq.RCVHWM, self.rcvhwm)
self.device.setsockopt_out(zmq.SNDHWM, self.outhwm)
self.device.bind_in(endpoint)
self.device.bind_out("ipc:///tmp/decin" + seed)
else:
self.device = Subscriber(endpoint, seed=seed)
self.decoder = Decoder(context,
procs=procs,
seed=seed,
rcvmeta=mapreduce)
self.started = False
def start_device(self):
queuedevice = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH)
queuedevice.bind_in(QUEUE_IN)
queuedevice.bind_out(QUEUE_OUT)
#queuedevice.setsockopt_in(zmq.HWM, 1)
#queuedevice.setsockopt_out(zmq.HWM, 1)
queuedevice.start()
def __init__(self, zmq_in=None, zmq_out=None):
'''
Instanciate the object
In :
zmq_in : zmq address:port combo for input (SUB)
zmq_out : zmq address:port combo for output (PUB)
'''
if (zmq_in == None):
self.zmq_bridge_in = 'tcp://127.0.0.1:42569'
else:
self.zmq_bridge_in = zmq_in
if (zmq_out == None):
self.zmq_bridge_out = 'tcp://127.0.0.1:42568'
else:
self.zmq_bridge_out = zmq_out
self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB)
self._processd.bind_in(self.zmq_bridge_in)
self._processd.bind_out(self.zmq_bridge_out)
self._processd.setsockopt_in(zmq.SUBSCRIBE, b'')
self._ident = 254
self._ident_number = 1
class Plug(object):
'''
Plug class to bridge the signal.
Create an instance zmq.ProcessDevice SUB -> QUEUE -> PUB
SUB socket has no filter
'''
def __init__(self, zmq_in=None, zmq_out=None):
'''
Instanciate the object
In :
zmq_in : zmq address:port combo for input (SUB)
zmq_out : zmq address:port combo for output (PUB)
'''
if (zmq_in == None):
self.zmq_bridge_in = 'tcp://127.0.0.1:42569'
else:
self.zmq_bridge_in = zmq_in
if (zmq_out == None):
self.zmq_bridge_out = 'tcp://127.0.0.1:42568'
else:
self.zmq_bridge_out = zmq_out
self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB)
self._processd.bind_in(self.zmq_bridge_in)
self._processd.bind_out(self.zmq_bridge_out)
self._processd.setsockopt_in(zmq.SUBSCRIBE, b'')
self._ident = 254
self._ident_number = 1
def plug_info(self, increment=True):
if (increment):
self._ident_number = self._ident_number + 1
return (self.zmq_bridge_in, self.zmq_bridge_out,
self._ident_number - 1)
def start(self):
self._processd.start()
@staticmethod
def server_forever():
while (True):
try:
sleep(1)
except (KeyboardInterrupt, SystemExit):
break
class Plug(object):
'''
Plug class to bridge the signal.
Create an instance zmq.ProcessDevice SUB -> QUEUE -> PUB
SUB socket has no filter
'''
def __init__(self, zmq_in = None, zmq_out = None):
'''
Instanciate the object
In :
zmq_in : zmq address:port combo for input (SUB)
zmq_out : zmq address:port combo for output (PUB)
'''
if(zmq_in == None):
self.zmq_bridge_in = 'tcp://127.0.0.1:42569'
else:
self.zmq_bridge_in = zmq_in
if(zmq_out == None):
self.zmq_bridge_out = 'tcp://127.0.0.1:42568'
else:
self.zmq_bridge_out = zmq_out
self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB)
self._processd.bind_in(self.zmq_bridge_in)
self._processd.bind_out(self.zmq_bridge_out)
self._processd.setsockopt_in(zmq.SUBSCRIBE,b'')
self._ident = 254
self._ident_number = 1
def plug_info(self, increment = True):
if(increment):
self._ident_number = self._ident_number + 1
return (self.zmq_bridge_in, self.zmq_bridge_out, self._ident_number - 1)
def start(self):
self._processd.start()
@staticmethod
def server_forever():
while(True):
try:
sleep(1)
except(KeyboardInterrupt, SystemExit):
break
def __init__(self, zmq_in = None, zmq_out = None):
'''
Instanciate the object
In :
zmq_in : zmq address:port combo for input (SUB)
zmq_out : zmq address:port combo for output (PUB)
'''
if(zmq_in == None):
self.zmq_bridge_in = 'tcp://127.0.0.1:42569'
else:
self.zmq_bridge_in = zmq_in
if(zmq_out == None):
self.zmq_bridge_out = 'tcp://127.0.0.1:42568'
else:
self.zmq_bridge_out = zmq_out
self._processd = ProcessDevice(zmq.QUEUE, zmq.SUB, zmq.PUB)
self._processd.bind_in(self.zmq_bridge_in)
self._processd.bind_out(self.zmq_bridge_out)
self._processd.setsockopt_in(zmq.SUBSCRIBE,b'')
self._ident = 254
self._ident_number = 1
def __init__(self, args):
self.args = args
self.context = zmq.Context()
self.sock = self.context.socket(zmq.REP)
self.pool = set(["Opp2Bot", "OppBotBattle", "OppBotBattleNoRaven", \
"OppBotTank", "OppBotBanshee", "OppBotFlashNew", "OppBotFlashNewRaven", "OppBotFlashFast", "OppBotMagic", "1", "2", "3", "4"])
self.win = dict()
for p in self.pool:
self.win[p] = []
if platform.system() == 'Windows':
# Windows에서는 Process기반 queue device 실행
# 다른 프로토콜 사용 불가능
from zmq.devices import ProcessDevice
device = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
device.bind_in(f"tcp://*:{args.frontend_port}")
device.bind_out(f"tcp://*:{args.backend_port}")
device.setsockopt_in(zmq.IDENTITY, 'ROUTER'.encode())
device.setsockopt_out(zmq.IDENTITY, 'DEALER'.encode())
device.start()
self.sock.connect("tcp://localhost:{}".format(args.backend_port))
else:
# Linux에서는 backend protocol을 tcp보다 빠른
# inproc (inter-process)를 사용하고,
# thread 기반 queue device를 실행
import threading
_device = threading.Thread(
target=device_func,
args=(self.context, f"tcp://*:{args.frontend_port}",
f"inproc://backend"),
)
_device.start()
self.sock.connect(f"inproc://backend")
def run_zmq_server():
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
zmq_in = f"ipc:///tmp/pd_in_{int(time())}_{randint(0, 1000)}"
zmq_out = f"ipc:///tmp/pd_in_{int(time())}_{randint(0, 1000)}"
streamerdevice.bind_in(zmq_in)
streamerdevice.bind_out(zmq_out)
streamerdevice.setsockopt_in(zmq.IDENTITY, b'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, b'PUSH')
streamerdevice.start()
return streamerdevice, zmq_in, zmq_out
# -*- coding: utf-8 -*-
import zmq
from zmq.devices import ProcessDevice
pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
pd.bind_in('tcp://*:12345')
pd.connect_out('tcp://127.0.0.1:5000')
pd.setsockopt_in(zmq.IDENTITY, 'ROUTER')
pd.setsockopt_out(zmq.IDENTITY, 'DEALER')
pd.start()
pd.join()
def __init__(self, args):
self.args = args
self.context = zmq.Context()
self.sock = self.context.socket(zmq.REP)
if platform.system() == 'Windows':
# Windows에서는 Process기반 queue device 실행
# 다른 프로토콜 사용 불가능
from zmq.devices import ProcessDevice
device = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
device.bind_in(f"tcp://*:{args.frontend_port}")
device.bind_out(f"tcp://*:{args.backend_port}")
device.setsockopt_in(zmq.IDENTITY, 'ROUTER'.encode())
device.setsockopt_out(zmq.IDENTITY, 'DEALER'.encode())
device.start()
self.sock.connect("tcp://localhost:{}".format(args.backend_port))
else:
# Linux에서는 backend protocol을 tcp보다 빠른
# inproc (inter-process)를 사용하고,
# thread 기반 queue device를 실행
import threading
_device = threading.Thread(
target=device_func,
args=(self.context, f"tcp://*:{args.frontend_port}",
f"inproc://backend"),
)
_device.start()
self.sock.connect(f"inproc://backend")
def forwarder_device(port_in, port_out):
from zmq.devices import ProcessDevice
pd = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
pd.bind_in('tcp://*:%s' % port_in)
pd.bind_out('tcp://*:%s' % port_out)
pd.setsockopt_in(zmq.IDENTITY, 'SUB')
pd.setsockopt_in(zmq.SUBSCRIBE, "")
pd.setsockopt_out(zmq.IDENTITY, 'PUB')
pd.start()
def streamer_device(port_in, port_out):
from zmq.devices import ProcessDevice
pd = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH)
pd.bind_in('tcp://*:%s' % port_in)
pd.bind_out('tcp://*:%s' % port_out)
pd.setsockopt_in(zmq.IDENTITY, 'PULL')
pd.setsockopt_out(zmq.IDENTITY, 'PUSH')
pd.start()
class Server(object):
def __init__(self, db, client_uri, internal_uri, lock_uri):
self.client_uri = client_uri
self.internal_uri = internal_uri
self.rep_uri = internal_uri.replace("*", "localhost")
self.lock_uri = lock_uri
self.db = db
self.pd = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.pd.bind_in(self.client_uri)
self.pd.bind_out(self.internal_uri)
self.pd.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.pd.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.running = False
def start(self):
self.pd.start()
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REP)
self.socket.connect(self.rep_uri)
def run(self):
self.running = True
while self.running:
message = self.socket.recv()
logging.debug("Received request: %s" % message)
try:
message = json.loads(message)
if message["mode"] == "exec":
# Make extra checks here
old_stdout = sys.stdout
stdout = sys.stdout = StringIO()
try:
co = compile(message["command"], "<remote>", "single")
exec co in globals()
except:
output = sys.exc_info()
else:
output = stdout.getvalue()
sys.stdout = old_stdout
elif message["mode"] == "readall":
output = db
elif message["mode"] == "lock":
output = {"locked": True, "uri": self.lock_uri}
elif message["mode"] == "unlock":
output = {"locked": False}
else:
entry = self.db
for key in message["index"]:
entry = entry[key]
if not message.get("func"):
output = entry
else:
func = getattr(entry, message["func"])
message["args"] = deepwrap(message["args"], entry.callbacks, entry.undocallbacks, self.db.wrapfile, skiproot=True)
message["kwargs"] = deepwrap(message["kwargs"], entry.callbacks, entry.undocallbacks, self.db.wrapfile, skiproot=True)
output = func(*message["args"], **message["kwargs"])
except:
output = traceback.format_exc()
logging.error(traceback.print_exc())
if type(output).__name__ in ['listiterator', 'dictionary-keyiterator']:
output = list(output)
try:
output = json.dumps(output, cls=Encoder)
except:
output = str(output)
self.socket.send(output)
if message["mode"] == "lock":
self.normal_socket = self.socket
self.socket = zmq.Context().socket(zmq.REP)
self.socket.bind(self.lock_uri.replace("localhost", "*"))
logging.debug("Locked and listening on %s" % self.lock_uri)
elif message["mode"] == "unlock":
self.socket.close()
self.socket = self.normal_socket
logging.debug("Unlocked")
def forwarder_device(port_in, port_out):
from zmq.devices import ProcessDevice
pd = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
pd.bind_in('tcp://*:%s' % port_in)
pd.bind_out('tcp://*:%s' % port_out)
pd.setsockopt_in(zmq.IDENTITY, 'SUB')
pd.setsockopt_in(zmq.SUBSCRIBE, "")
pd.setsockopt_out(zmq.IDENTITY, 'PUB')
pd.start()
def streamer_device(port_in, port_out):
from zmq.devices import ProcessDevice
pd = ProcessDevice(zmq.QUEUE, zmq.PULL, zmq.PUSH)
pd.bind_in('tcp://*:%s' % port_in)
pd.bind_out('tcp://*:%s' % port_out)
pd.setsockopt_in(zmq.IDENTITY, 'PULL')
pd.setsockopt_out(zmq.IDENTITY, 'PUSH')
pd.start()
context = zmq.Context()
client_socket = context.socket(zmq.REQ)
client_socket.connect("tcp://%s:%s" % (config['client_ip'], config['client_port']))
def default(self, line):
line = str(line)
self.do_send(line)
def do_send(self, line):
self.client_socket.send_string(line)
msg = self.client_socket.recv_string()
print msg
def do_receive(self, line):
"""
Should be able to remove this.
"""
self.client_socket.recv_string()
def do_debug(self, line):
ipdb.set_trace()
if __name__ == '__main__':
queue_process = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
queue_process.connect_out("tcp://%s:%s" % (config['server_ip'], config['server_port']))
queue_process.bind_in("tcp://%s:%s" % (config['client_ip'], config['client_port']))
queue_process.start()
CLI().cmdloop(intro='Welcome to the shell, mortal.')
