def __init__(self, device_connection=None, pub_port=None, sub_port=None): ''' Set up the pub_sub_handler socket connections Parameters: device_connection: The tcp IP of the mechoscore to connect to. Default is tcp://127.0.0.101 pub_port:The tcp socket that publishers will connect to. Default 5559 sub_port: The tcp socket that subscribers will connect to. Default 5560 ''' if device_connection is None: device_connection = "127.0.0.101" if pub_port is None: pub_port = "5559" if sub_port is None: sub_port = "5560" self._device_connection = "tcp://" + device_connection self._pub_port = pub_port self._sub_port = sub_port self._pub_connection_socket = self._device_connection + ":" + pub_port self._sub_connection_socket = self._device_connection + ":" + sub_port #device running as an isolated process that routes pub/sub messages self._pub_sub_handler_device = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB) self._pub_sub_handler_device.bind_in(self._pub_connection_socket) self._pub_sub_handler_device.bind_out(self._sub_connection_socket) self._pub_sub_handler_device.setsockopt_in(zmq.SUBSCRIBE, "".encode("utf-8"))
class _Pub_Sub_Handler: ''' Communication link for publisher/subscribers. Utilize zmq devices to to automatically rout publisher messages to subscribers of the same topic name. ''' def __init__(self, device_connection=None, pub_port=None, sub_port=None): ''' Set up the pub_sub_handler socket connections Parameters: device_connection: The tcp IP of the mechoscore to connect to. Default is tcp://127.0.0.101 pub_port:The tcp socket that publishers will connect to. Default 5559 sub_port: The tcp socket that subscribers will connect to. Default 5560 ''' if device_connection is None: device_connection = "127.0.0.101" if pub_port is None: pub_port = "5559" if sub_port is None: sub_port = "5560" self._device_connection = "tcp://" + device_connection self._pub_port = pub_port self._sub_port = sub_port self._pub_connection_socket = self._device_connection + ":" + pub_port self._sub_connection_socket = self._device_connection + ":" + sub_port #device running as an isolated process that routes pub/sub messages self._pub_sub_handler_device = ProcessDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB) self._pub_sub_handler_device.bind_in(self._pub_connection_socket) self._pub_sub_handler_device.bind_out(self._sub_connection_socket) self._pub_sub_handler_device.setsockopt_in(zmq.SUBSCRIBE, "".encode("utf-8")) def start_pub_sub_handler(self): ''' Start the isolated process that routes publisher messages to subscribers This process will be terminated once python script terminates Parameters: N/A Returns: N/A ''' print("Publisher connection socket location:", self._pub_connection_socket) print("Subscriber conneciton socket location:", self._sub_connection_socket) self._pub_sub_handler_device.start()
def create_streamer(pull_port, push_port): """Vytvoření streameru.""" streamer_device = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) frontend_address = "tcp://*:{port}".format(port=pull_port) backend_address = "tcp://*:{port}".format(port=push_port) streamer_device.bind_in(frontend_address) streamer_device.bind_out(backend_address) print("Bound to {a} on port {p}".format(a=frontend_address, p=pull_port)) print("Bound to {a} on port {p}".format(a=backend_address, p=push_port)) streamer_device.start() print("Device started in background")
def create_queue(xrep_port, xreq_port): """Vytvoření fronty.""" queue_device = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) frontend_address = "tcp://*:{port}".format(port=xrep_port) backend_address = "tcp://*:{port}".format(port=xreq_port) queue_device.bind_in(frontend_address) queue_device.bind_out(backend_address) print("Bound to {a} on port {p}".format(a=frontend_address, p=xrep_port)) print("Bound to {a} on port {p}".format(a=backend_address, p=xreq_port)) queue_device.start() print("Queue device started in background")
def prepare(self): self.balancer = None if options.worker_bind_address: self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.balancer.daemon = True self.balancer.bind_in(options.worker_bind_address) self.balancer.bind_out(options.worker_socket_address) self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER') self.balancer.start() if options.daemon: with open(pid_path(0), 'wb') as f: f.write(str(self.balancer.launcher.pid)) count = options.processes if options.processes >= 0 else cpu_count() if count == 0: print 'Starting load balancer. Listening on "%s". Routing to "%s"' % ( options.worker_bind_address, options.worker_socket_address) else: print "Starting %s worker process%s. %s." % ( count, count > 1 and 'es' or '', options.worker_bind_address and ('Listening on "%s"' % options.worker_bind_address) or ('Connecting to "%s"' % options.worker_socket_address))
def prepare(self): self.balancer = None if options.worker_address: self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.balancer.daemon = True self.balancer.bind_in(options.worker_address) self.balancer.bind_out(options.worker_socket_address) self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER') self.balancer.start() if options.daemon: with open(pid_path(0), 'wb') as f: f.write(str(self.balancer.launcher.pid)) count = options.processes if options.processes >= 0 else cpu_count() if count == 0: print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address) else: print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address))
import time import zmq from zmq.devices.basedevice import ProcessDevice pd = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUB) pd.bind_in("tcp://*:6767") pd.connect_out("epgm://eth0;239.192.0.1:7601") pd.setsockopt_out(zmq.RATE, 10000) pd.start() # Do other things here # This is just to pretend do some foreground work. while True: time.sleep(100)
def init_streamer(frontend_port, backend_port): streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port) streamerdevice.setsockopt_in(zmq.IDENTITY, "PULL".encode('utf-8')) streamerdevice.setsockopt_out(zmq.IDENTITY, "PUSH".encode('utf-8')) streamerdevice.start()
import time import zmq from zmq.devices.basedevice import ProcessDevice from multiprocessing import Process frontend_port = 5559 backend_port = 5560 number_of_workers = 5 streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port ) streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port) streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL') streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH') streamerdevice.start() def server(): context = zmq.Context() socket = context.socket(zmq.PUSH) socket.connect("tcp://127.0.0.1:%d" % frontend_port) for i in xrange(0,100): socket.send('#%s' % i) def worker(work_num): context = zmq.Context() socket = context.socket(zmq.PULL) socket.connect("tcp://127.0.0.1:%d" % backend_port)
def streamer_device(frontend_port, backend_port): frontend_address = "tcp://127.0.0.1:%d" % frontend_port backend_address = "tcp://127.0.0.1:%d" % frontend_port streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) streamerdevice.bind_in(frontend_address) streamerdevice.bind_out(backend_address) streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL') streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH') streamerdevice.start() click.echo("started streamer on backend: %s, frontend: %s" %( backend_address, frontend_address))
class TotoWorkerService(TotoService): def __init__(self, conf_file=None, **kwargs): module_options = {'method_module', 'event_init_module'} function_options = {'startup_function'} original_argv, sys.argv = sys.argv, [i for i in sys.argv if i.strip('-').split('=')[0] in module_options] self._load_options(conf_file, **{i: kwargs[i] for i in kwargs if i in module_options}) modules = {getattr(options, i) for i in module_options if getattr(options, i)} for module in modules: __import__(module) function_modules = {getattr(options, i).rsplit('.', 1)[0] for i in function_options if getattr(options, i)} for module in function_modules: __import__(module) sys.argv = original_argv #clear root logger handlers to prevent duplicate logging if user has specified a log file super(TotoWorkerService, self).__init__(conf_file, **kwargs) #clear method_module references so we can fully reload with new options for module in modules: for i in (m for m in sys.modules.keys() if m.startswith(module)): del sys.modules[i] for module in function_modules: for i in (m for m in sys.modules.keys() if m.startswith(module)): del sys.modules[i] #prevent the reloaded module from re-defining options define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None self.__event_init = options.event_init_module and __import__(options.event_init_module) or None self.__method_module = options.method_module and __import__(options.method_module) or None tornado.options.define = define def prepare(self): self.balancer = None if options.worker_address: self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.balancer.daemon = True self.balancer.bind_in(options.worker_address) self.balancer.bind_out(options.worker_socket_address) self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER') self.balancer.start() if options.daemon: with open(pid_path(0), 'wb') as f: f.write(str(self.balancer.launcher.pid)) count = options.processes if options.processes >= 0 else cpu_count() if count == 0: print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address) else: print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address)) def main_loop(self): db_connection = configured_connection() if options.remote_event_receivers: from toto.events import EventManager event_manager = EventManager.instance() if options.remote_instances: for address in options.remote_event_receivers.split(','): event_manager.register_server(address) init_module = self.__event_init if init_module: init_module.invoke(event_manager) serialization = options.serialization_module and __import__(options.serialization_module) or pickle compression = options.compression_module and __import__(options.compression_module) worker = TotoWorker(self.__method_module, options.worker_socket_address, db_connection, compression, serialization) if options.startup_function: startup_path = options.startup_function.rsplit('.') __import__(startup_path[0]).__dict__[startup_path[1]](worker=worker, db_connection=db_connection) worker.start() def send_worker_command(self, command): if options.control_socket_address: socket = zmq.Context().socket(zmq.PUB) socket.bind(options.control_socket_address) time.sleep(1) socket.send_string('command %s' % command) print "Sent command: %s" % options.command def run(self): if options.command: self.send_worker_command(options.command) return super(TotoWorkerService, self).run()
import time import zmq from zmq.devices.basedevice import ProcessDevice from multiprocessing import Process import random frontend_port = 5559 backend_port = 5560 number_of_workers = 2 queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port) queuedevice.setsockopt_in(zmq.HWM, 1) queuedevice.setsockopt_out(zmq.HWM, 1) queuedevice.start() time.sleep (2) def server(backend_port): print "Connecting a server to queue device" context = zmq.Context() socket = context.socket(zmq.REP) socket.connect("tcp://127.0.0.1:%s" % backend_port) server_id = random.randrange(1,10005) while True: message = socket.recv() print "Received request: ", message socket.send("Response from %s" % server_id) def client(frontend_port, client_id):
import time import zmq from zmq.devices.basedevice import ProcessDevice import multiprocessing # what is this f**k all about? # f**k? if __name__ == "__main__": # pass" multiprocessing.freeze_support() frontend_port = 5559 backend_port = 5560 # number_of_workers = 2 # what the f**k? streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PUB, zmq.SUB) # d0 = dir(streamerdevice) """['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_in_binds', '_in_connects', '_in_sockopts', '_launch_class', '_out_binds', '_out_connects', '_out_sockopts', '_random_addrs', '_reserve_random_port', '_setup_sockets', 'bind_in', 'bind_in_to_random_port', 'bind_out', 'bind_out_to_random_port', 'connect_in', 'connect_out', 'context_factory', 'daemon', 'device_type', 'done', 'in_type', 'join', 'launcher', 'out_type', 'run', 'run_device', 'setsockopt_in', 'setsockopt_out', 'start']""" # print(d0) streamerdevice.bind_in("tcp://127.0.0.9:%d" % frontend_port) streamerdevice.bind_out("tcp://127.0.0.9:%d" % backend_port) # print(streamerdevice._out_sockopts) # we have got shit. # streamerdevice.setsockopt_string(zmq.IDENTITY, 'PUB') # streamerdevice.setsockopt_out(zmq.IDENTITY, 'SUB') streamerdevice.start() # this is f****d up. # f*****g shit. while True:
import time import zmq from zmq.devices.basedevice import ProcessDevice from multiprocessing import Process frontend_port = 5559 backend_port = 5560 number_of_workers = 2 streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port) streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL') streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH') streamerdevice.start() def server(): context = zmq.Context() socket = context.socket(zmq.PUSH) socket.connect("tcp://127.0.0.1:%d" % frontend_port) for i in xrange(0, 10): socket.send('#%s' % i) def worker(work_num): context = zmq.Context() socket = context.socket(zmq.PULL) socket.connect("tcp://127.0.0.1:%d" % backend_port)
import time import zmq from zmq.devices.basedevice import ProcessDevice from multiprocessing import Process import random frontend_port = 5559 backend_port = 5560 number_of_workers = 2 queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ) queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port) queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port) queuedevice.setsockopt_in(zmq.RCVHWM, 1) queuedevice.setsockopt_out(zmq.SNDHWM, 1) queuedevice.start() time.sleep (2) def server(backend_port): print "Connecting a server to queue device" context = zmq.Context() socket = context.socket(zmq.REP) socket.connect("tcp://127.0.0.1:%s" % backend_port) server_id = random.randrange(1,10005) while True: message = socket.recv() print "Received request: ", message socket.send("Response from %s" % server_id)
class TotoWorkerService(TotoService): '''Instances can be configured in three ways: 1. (Most common) Pass the path to a config file as the first parameter to the constructor. 2. Pass config parameters as command line arguments to the initialization script. 3. Pass keyword arguments to the constructor. Precidence is as follows: Keyword args, config file, command line ''' def __init__(self, conf_file=None, **kwargs): module_options = {'method_module', 'event_init_module'} function_options = {'startup_function'} original_argv, sys.argv = sys.argv, [ i for i in sys.argv if i.strip('-').split('=')[0] in module_options ] self._load_options( conf_file, **{i: kwargs[i] for i in kwargs if i in module_options}) modules = { getattr(options, i) for i in module_options if getattr(options, i) } for module in modules: __import__(module) function_modules = { getattr(options, i).rsplit('.', 1)[0] for i in function_options if getattr(options, i) } for module in function_modules: __import__(module) sys.argv = original_argv #clear root logger handlers to prevent duplicate logging if user has specified a log file super(TotoWorkerService, self).__init__(conf_file, **kwargs) #clear method_module references so we can fully reload with new options for module in modules: for i in (m for m in sys.modules.keys() if m.startswith(module)): del sys.modules[i] for module in function_modules: for i in (m for m in sys.modules.keys() if m.startswith(module)): del sys.modules[i] #prevent the reloaded module from re-defining options define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None self.__event_init = options.event_init_module and __import__( options.event_init_module) or None self.__method_module = options.method_module and __import__( options.method_module) or None tornado.options.define = define def prepare(self): self.balancer = None if options.worker_bind_address: self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER) self.balancer.daemon = True self.balancer.bind_in(options.worker_bind_address) self.balancer.bind_out(options.worker_socket_address) self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER') self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER') self.balancer.start() if options.daemon: with open(pid_path(0), 'wb') as f: f.write(str(self.balancer.launcher.pid)) count = options.processes if options.processes >= 0 else cpu_count() if count == 0: print 'Starting load balancer. Listening on "%s". Routing to "%s"' % ( options.worker_bind_address, options.worker_socket_address) else: print "Starting %s worker process%s. %s." % ( count, count > 1 and 'es' or '', options.worker_bind_address and ('Listening on "%s"' % options.worker_bind_address) or ('Connecting to "%s"' % options.worker_socket_address)) def main_loop(self): db_connection = configured_connection() if options.remote_event_receivers: from toto.events import EventManager event_manager = EventManager.instance() if options.remote_instances: for address in options.remote_event_receivers.split(','): event_manager.register_server(address) init_module = self.__event_init if init_module: init_module.invoke(event_manager) serialization = options.serialization_module and __import__( options.serialization_module) or pickle compression = options.compression_module and __import__( options.compression_module) worker = TotoWorker(self.__method_module, options.worker_socket_address, db_connection, compression, serialization) if options.startup_function: startup_path = options.startup_function.rsplit('.') __import__(startup_path[0]).__dict__[startup_path[1]]( worker=worker, db_connection=db_connection) worker.start() def send_worker_command(self, command): if options.control_socket_address: socket = zmq.Context().socket(zmq.PUB) socket.bind(options.control_socket_address) time.sleep(1) socket.send_string('command %s' % command) print "Sent command: %s" % options.command def run(self): if options.command: self.send_worker_command(options.command) return super(TotoWorkerService, self).run()