def init_streamer(frontend_port, backend_port):
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, "PULL".encode('utf-8'))
streamerdevice.setsockopt_out(zmq.IDENTITY, "PUSH".encode('utf-8'))
streamerdevice.start()
def streamer_device(frontend_port, backend_port):
frontend_address = "tcp://127.0.0.1:%d" % frontend_port
backend_address = "tcp://127.0.0.1:%d" % frontend_port
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in(frontend_address)
streamerdevice.bind_out(backend_address)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
click.echo("started streamer on backend: %s, frontend: %s" %(
backend_address, frontend_address))
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
import random
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
queuedevice.setsockopt_in(zmq.RCVHWM, 1)
queuedevice.setsockopt_out(zmq.SNDHWM, 1)
queuedevice.start()
time.sleep (2)
def server(backend_port):
print "Connecting a server to queue device"
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.connect("tcp://127.0.0.1:%s" % backend_port)
server_id = random.randrange(1,10005)
while True:
message = socket.recv()
print "Received request: ", message
socket.send("Response from %s" % server_id)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
def server():
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in xrange(0, 10):
socket.send('#%s' % i)
def worker(work_num):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
pd = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUB)
pd.bind_in("tcp://*:6767")
pd.connect_out("epgm://eth0;239.192.0.1:7601")
pd.setsockopt_out(zmq.RATE, 10000)
pd.start()
# Do other things here
# This is just to pretend do some foreground work.
while True:
time.sleep(100)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
frontend_port = 5559
backend_port = 5560
number_of_workers = 5
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port )
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
def server():
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in xrange(0,100):
socket.send('#%s' % i)
def worker(work_num):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
pd = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUB)
pd.bind_in("tcp://*:6767")
pd.connect_out("epgm://eth0;239.192.0.1:7601")
pd.setsockopt_out(zmq.RATE, 10000)
pd.start()
# Do other things here
# This is just to pretend do some foreground work.
while True:
time.sleep(100)
class TotoWorkerService(TotoService):
def __init__(self, conf_file=None, **kwargs):
module_options = {'method_module', 'event_init_module'}
function_options = {'startup_function'}
original_argv, sys.argv = sys.argv, [i for i in sys.argv if i.strip('-').split('=')[0] in module_options]
self._load_options(conf_file, **{i: kwargs[i] for i in kwargs if i in module_options})
modules = {getattr(options, i) for i in module_options if getattr(options, i)}
for module in modules:
__import__(module)
function_modules = {getattr(options, i).rsplit('.', 1)[0] for i in function_options if getattr(options, i)}
for module in function_modules:
__import__(module)
sys.argv = original_argv
#clear root logger handlers to prevent duplicate logging if user has specified a log file
super(TotoWorkerService, self).__init__(conf_file, **kwargs)
#clear method_module references so we can fully reload with new options
for module in modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
for module in function_modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
#prevent the reloaded module from re-defining options
define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None
self.__event_init = options.event_init_module and __import__(options.event_init_module) or None
self.__method_module = options.method_module and __import__(options.method_module) or None
tornado.options.define = define
def prepare(self):
self.balancer = None
if options.worker_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address))
def main_loop(self):
db_connection = configured_connection()
if options.remote_event_receivers:
from toto.events import EventManager
event_manager = EventManager.instance()
if options.remote_instances:
for address in options.remote_event_receivers.split(','):
event_manager.register_server(address)
init_module = self.__event_init
if init_module:
init_module.invoke(event_manager)
serialization = options.serialization_module and __import__(options.serialization_module) or pickle
compression = options.compression_module and __import__(options.compression_module)
worker = TotoWorker(self.__method_module, options.worker_socket_address, db_connection, compression, serialization)
if options.startup_function:
startup_path = options.startup_function.rsplit('.')
__import__(startup_path[0]).__dict__[startup_path[1]](worker=worker, db_connection=db_connection)
worker.start()
def send_worker_command(self, command):
if options.control_socket_address:
socket = zmq.Context().socket(zmq.PUB)
socket.bind(options.control_socket_address)
time.sleep(1)
socket.send_string('command %s' % command)
print "Sent command: %s" % options.command
def run(self):
if options.command:
self.send_worker_command(options.command)
return
super(TotoWorkerService, self).run()
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
import random
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
queuedevice.setsockopt_in(zmq.HWM, 1)
queuedevice.setsockopt_out(zmq.HWM, 1)
queuedevice.start()
time.sleep (2)
def server(backend_port):
print "Connecting a server to queue device"
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.connect("tcp://127.0.0.1:%s" % backend_port)
server_id = random.randrange(1,10005)
while True:
message = socket.recv()
print "Received request: ", message
socket.send("Response from %s" % server_id)
def client(frontend_port, client_id):
class TotoWorkerService(TotoService):
'''Instances can be configured in three ways:
1. (Most common) Pass the path to a config file as the first parameter to the constructor.
2. Pass config parameters as command line arguments to the initialization script.
3. Pass keyword arguments to the constructor.
Precidence is as follows:
Keyword args, config file, command line
'''
def __init__(self, conf_file=None, **kwargs):
module_options = {'method_module', 'event_init_module'}
function_options = {'startup_function'}
original_argv, sys.argv = sys.argv, [
i for i in sys.argv if i.strip('-').split('=')[0] in module_options
]
self._load_options(
conf_file, **{i: kwargs[i]
for i in kwargs if i in module_options})
modules = {
getattr(options, i)
for i in module_options if getattr(options, i)
}
for module in modules:
__import__(module)
function_modules = {
getattr(options, i).rsplit('.', 1)[0]
for i in function_options if getattr(options, i)
}
for module in function_modules:
__import__(module)
sys.argv = original_argv
#clear root logger handlers to prevent duplicate logging if user has specified a log file
super(TotoWorkerService, self).__init__(conf_file, **kwargs)
#clear method_module references so we can fully reload with new options
for module in modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
for module in function_modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
#prevent the reloaded module from re-defining options
define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None
self.__event_init = options.event_init_module and __import__(
options.event_init_module) or None
self.__method_module = options.method_module and __import__(
options.method_module) or None
tornado.options.define = define
def prepare(self):
self.balancer = None
if options.worker_bind_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_bind_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (
options.worker_bind_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (
count, count > 1 and 'es'
or '', options.worker_bind_address and
('Listening on "%s"' % options.worker_bind_address) or
('Connecting to "%s"' % options.worker_socket_address))
def main_loop(self):
db_connection = configured_connection()
if options.remote_event_receivers:
from toto.events import EventManager
event_manager = EventManager.instance()
if options.remote_instances:
for address in options.remote_event_receivers.split(','):
event_manager.register_server(address)
init_module = self.__event_init
if init_module:
init_module.invoke(event_manager)
serialization = options.serialization_module and __import__(
options.serialization_module) or pickle
compression = options.compression_module and __import__(
options.compression_module)
worker = TotoWorker(self.__method_module,
options.worker_socket_address, db_connection,
compression, serialization)
if options.startup_function:
startup_path = options.startup_function.rsplit('.')
__import__(startup_path[0]).__dict__[startup_path[1]](
worker=worker, db_connection=db_connection)
worker.start()
def send_worker_command(self, command):
if options.control_socket_address:
socket = zmq.Context().socket(zmq.PUB)
socket.bind(options.control_socket_address)
time.sleep(1)
socket.send_string('command %s' % command)
print "Sent command: %s" % options.command
def run(self):
if options.command:
self.send_worker_command(options.command)
return
super(TotoWorkerService, self).run()
