def __init__(self, device_connection=None, pub_port=None,
sub_port=None):
'''
Set up the pub_sub_handler socket connections
Parameters:
device_connection: The tcp IP of the mechoscore to connect to.
Default is tcp://127.0.0.101
pub_port:The tcp socket that publishers will connect to. Default 5559
sub_port: The tcp socket that subscribers will connect to. Default 5560
'''
if device_connection is None:
device_connection = "127.0.0.101"
if pub_port is None:
pub_port = "5559"
if sub_port is None:
sub_port = "5560"
self._device_connection = "tcp://" + device_connection
self._pub_port = pub_port
self._sub_port = sub_port
self._pub_connection_socket = self._device_connection + ":" + pub_port
self._sub_connection_socket = self._device_connection + ":" + sub_port
#device running as an isolated process that routes pub/sub messages
self._pub_sub_handler_device = ProcessDevice(zmq.FORWARDER, zmq.SUB,
zmq.PUB)
self._pub_sub_handler_device.bind_in(self._pub_connection_socket)
self._pub_sub_handler_device.bind_out(self._sub_connection_socket)
self._pub_sub_handler_device.setsockopt_in(zmq.SUBSCRIBE,
"".encode("utf-8"))
class _Pub_Sub_Handler:
'''
Communication link for publisher/subscribers. Utilize zmq devices to
to automatically rout publisher messages to subscribers of the same topic
name.
'''
def __init__(self, device_connection=None, pub_port=None,
sub_port=None):
'''
Set up the pub_sub_handler socket connections
Parameters:
device_connection: The tcp IP of the mechoscore to connect to.
Default is tcp://127.0.0.101
pub_port:The tcp socket that publishers will connect to. Default 5559
sub_port: The tcp socket that subscribers will connect to. Default 5560
'''
if device_connection is None:
device_connection = "127.0.0.101"
if pub_port is None:
pub_port = "5559"
if sub_port is None:
sub_port = "5560"
self._device_connection = "tcp://" + device_connection
self._pub_port = pub_port
self._sub_port = sub_port
self._pub_connection_socket = self._device_connection + ":" + pub_port
self._sub_connection_socket = self._device_connection + ":" + sub_port
#device running as an isolated process that routes pub/sub messages
self._pub_sub_handler_device = ProcessDevice(zmq.FORWARDER, zmq.SUB,
zmq.PUB)
self._pub_sub_handler_device.bind_in(self._pub_connection_socket)
self._pub_sub_handler_device.bind_out(self._sub_connection_socket)
self._pub_sub_handler_device.setsockopt_in(zmq.SUBSCRIBE,
"".encode("utf-8"))
def start_pub_sub_handler(self):
'''
Start the isolated process that routes publisher messages to subscribers
This process will be terminated once python script terminates
Parameters:
N/A
Returns:
N/A
'''
print("Publisher connection socket location:", self._pub_connection_socket)
print("Subscriber conneciton socket location:",
self._sub_connection_socket)
self._pub_sub_handler_device.start()
def create_streamer(pull_port, push_port):
"""Vytvoření streameru."""
streamer_device = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
frontend_address = "tcp://*:{port}".format(port=pull_port)
backend_address = "tcp://*:{port}".format(port=push_port)
streamer_device.bind_in(frontend_address)
streamer_device.bind_out(backend_address)
print("Bound to {a} on port {p}".format(a=frontend_address, p=pull_port))
print("Bound to {a} on port {p}".format(a=backend_address, p=push_port))
streamer_device.start()
print("Device started in background")
def create_queue(xrep_port, xreq_port):
"""Vytvoření fronty."""
queue_device = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
frontend_address = "tcp://*:{port}".format(port=xrep_port)
backend_address = "tcp://*:{port}".format(port=xreq_port)
queue_device.bind_in(frontend_address)
queue_device.bind_out(backend_address)
print("Bound to {a} on port {p}".format(a=frontend_address, p=xrep_port))
print("Bound to {a} on port {p}".format(a=backend_address, p=xreq_port))
queue_device.start()
print("Queue device started in background")
def prepare(self):
self.balancer = None
if options.worker_bind_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_bind_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (
options.worker_bind_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (
count, count > 1 and 'es'
or '', options.worker_bind_address and
('Listening on "%s"' % options.worker_bind_address) or
('Connecting to "%s"' % options.worker_socket_address))
def prepare(self):
self.balancer = None
if options.worker_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address))
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
pd = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUB)
pd.bind_in("tcp://*:6767")
pd.connect_out("epgm://eth0;239.192.0.1:7601")
pd.setsockopt_out(zmq.RATE, 10000)
pd.start()
# Do other things here
# This is just to pretend do some foreground work.
while True:
time.sleep(100)
def init_streamer(frontend_port, backend_port):
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, "PULL".encode('utf-8'))
streamerdevice.setsockopt_out(zmq.IDENTITY, "PUSH".encode('utf-8'))
streamerdevice.start()
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
frontend_port = 5559
backend_port = 5560
number_of_workers = 5
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port )
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
def server():
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in xrange(0,100):
socket.send('#%s' % i)
def worker(work_num):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
pd = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUB)
pd.bind_in("tcp://*:6767")
pd.connect_out("epgm://eth0;239.192.0.1:7601")
pd.setsockopt_out(zmq.RATE, 10000)
pd.start()
# Do other things here
# This is just to pretend do some foreground work.
while True:
time.sleep(100)
def streamer_device(frontend_port, backend_port):
frontend_address = "tcp://127.0.0.1:%d" % frontend_port
backend_address = "tcp://127.0.0.1:%d" % frontend_port
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in(frontend_address)
streamerdevice.bind_out(backend_address)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
click.echo("started streamer on backend: %s, frontend: %s" %(
backend_address, frontend_address))
class TotoWorkerService(TotoService):
def __init__(self, conf_file=None, **kwargs):
module_options = {'method_module', 'event_init_module'}
function_options = {'startup_function'}
original_argv, sys.argv = sys.argv, [i for i in sys.argv if i.strip('-').split('=')[0] in module_options]
self._load_options(conf_file, **{i: kwargs[i] for i in kwargs if i in module_options})
modules = {getattr(options, i) for i in module_options if getattr(options, i)}
for module in modules:
__import__(module)
function_modules = {getattr(options, i).rsplit('.', 1)[0] for i in function_options if getattr(options, i)}
for module in function_modules:
__import__(module)
sys.argv = original_argv
#clear root logger handlers to prevent duplicate logging if user has specified a log file
super(TotoWorkerService, self).__init__(conf_file, **kwargs)
#clear method_module references so we can fully reload with new options
for module in modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
for module in function_modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
#prevent the reloaded module from re-defining options
define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None
self.__event_init = options.event_init_module and __import__(options.event_init_module) or None
self.__method_module = options.method_module and __import__(options.method_module) or None
tornado.options.define = define
def prepare(self):
self.balancer = None
if options.worker_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (options.worker_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (count, count > 1 and 'es' or '', options.worker_address and ('Listening on "%s"' % options.worker_address) or ('Connecting to "%s"' % options.worker_socket_address))
def main_loop(self):
db_connection = configured_connection()
if options.remote_event_receivers:
from toto.events import EventManager
event_manager = EventManager.instance()
if options.remote_instances:
for address in options.remote_event_receivers.split(','):
event_manager.register_server(address)
init_module = self.__event_init
if init_module:
init_module.invoke(event_manager)
serialization = options.serialization_module and __import__(options.serialization_module) or pickle
compression = options.compression_module and __import__(options.compression_module)
worker = TotoWorker(self.__method_module, options.worker_socket_address, db_connection, compression, serialization)
if options.startup_function:
startup_path = options.startup_function.rsplit('.')
__import__(startup_path[0]).__dict__[startup_path[1]](worker=worker, db_connection=db_connection)
worker.start()
def send_worker_command(self, command):
if options.control_socket_address:
socket = zmq.Context().socket(zmq.PUB)
socket.bind(options.control_socket_address)
time.sleep(1)
socket.send_string('command %s' % command)
print "Sent command: %s" % options.command
def run(self):
if options.command:
self.send_worker_command(options.command)
return
super(TotoWorkerService, self).run()
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
import random
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
queuedevice.setsockopt_in(zmq.HWM, 1)
queuedevice.setsockopt_out(zmq.HWM, 1)
queuedevice.start()
time.sleep (2)
def server(backend_port):
print "Connecting a server to queue device"
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.connect("tcp://127.0.0.1:%s" % backend_port)
server_id = random.randrange(1,10005)
while True:
message = socket.recv()
print "Received request: ", message
socket.send("Response from %s" % server_id)
def client(frontend_port, client_id):
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
import multiprocessing
# what is this f**k all about?
# f**k?
if __name__ == "__main__":
# pass"
multiprocessing.freeze_support()
frontend_port = 5559
backend_port = 5560
# number_of_workers = 2
# what the f**k?
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PUB, zmq.SUB)
# d0 = dir(streamerdevice)
"""['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__',
'__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_in_binds', '_in_connects', '_in_sockopts', '_launch_class', '_out_binds', '_out_connects',
'_out_sockopts', '_random_addrs', '_reserve_random_port', '_setup_sockets', 'bind_in', 'bind_in_to_random_port', 'bind_out', 'bind_out_to_random_port', 'connect_in', 'connect_out', 'context_factory', 'daemon', 'device_type', 'done', 'in_type', 'join', 'launcher', 'out_type', 'run', 'run_device', 'setsockopt_in', 'setsockopt_out', 'start']"""
# print(d0)
streamerdevice.bind_in("tcp://127.0.0.9:%d" % frontend_port)
streamerdevice.bind_out("tcp://127.0.0.9:%d" % backend_port)
# print(streamerdevice._out_sockopts)
# we have got shit.
# streamerdevice.setsockopt_string(zmq.IDENTITY, 'PUB')
# streamerdevice.setsockopt_out(zmq.IDENTITY, 'SUB')
streamerdevice.start()
# this is f****d up.
# f*****g shit.
while True:
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, 'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, 'PUSH')
streamerdevice.start()
def server():
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in xrange(0, 10):
socket.send('#%s' % i)
def worker(work_num):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
import random
frontend_port = 5559
backend_port = 5560
number_of_workers = 2
queuedevice = ProcessDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
queuedevice.bind_in("tcp://127.0.0.1:%d" % frontend_port)
queuedevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
queuedevice.setsockopt_in(zmq.RCVHWM, 1)
queuedevice.setsockopt_out(zmq.SNDHWM, 1)
queuedevice.start()
time.sleep (2)
def server(backend_port):
print "Connecting a server to queue device"
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.connect("tcp://127.0.0.1:%s" % backend_port)
server_id = random.randrange(1,10005)
while True:
message = socket.recv()
print "Received request: ", message
socket.send("Response from %s" % server_id)
class TotoWorkerService(TotoService):
'''Instances can be configured in three ways:
1. (Most common) Pass the path to a config file as the first parameter to the constructor.
2. Pass config parameters as command line arguments to the initialization script.
3. Pass keyword arguments to the constructor.
Precidence is as follows:
Keyword args, config file, command line
'''
def __init__(self, conf_file=None, **kwargs):
module_options = {'method_module', 'event_init_module'}
function_options = {'startup_function'}
original_argv, sys.argv = sys.argv, [
i for i in sys.argv if i.strip('-').split('=')[0] in module_options
]
self._load_options(
conf_file, **{i: kwargs[i]
for i in kwargs if i in module_options})
modules = {
getattr(options, i)
for i in module_options if getattr(options, i)
}
for module in modules:
__import__(module)
function_modules = {
getattr(options, i).rsplit('.', 1)[0]
for i in function_options if getattr(options, i)
}
for module in function_modules:
__import__(module)
sys.argv = original_argv
#clear root logger handlers to prevent duplicate logging if user has specified a log file
super(TotoWorkerService, self).__init__(conf_file, **kwargs)
#clear method_module references so we can fully reload with new options
for module in modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
for module in function_modules:
for i in (m for m in sys.modules.keys() if m.startswith(module)):
del sys.modules[i]
#prevent the reloaded module from re-defining options
define, tornado.options.define = tornado.options.define, lambda *args, **kwargs: None
self.__event_init = options.event_init_module and __import__(
options.event_init_module) or None
self.__method_module = options.method_module and __import__(
options.method_module) or None
tornado.options.define = define
def prepare(self):
self.balancer = None
if options.worker_bind_address:
self.balancer = ProcessDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
self.balancer.daemon = True
self.balancer.bind_in(options.worker_bind_address)
self.balancer.bind_out(options.worker_socket_address)
self.balancer.setsockopt_in(zmq.IDENTITY, 'ROUTER')
self.balancer.setsockopt_out(zmq.IDENTITY, 'DEALER')
self.balancer.start()
if options.daemon:
with open(pid_path(0), 'wb') as f:
f.write(str(self.balancer.launcher.pid))
count = options.processes if options.processes >= 0 else cpu_count()
if count == 0:
print 'Starting load balancer. Listening on "%s". Routing to "%s"' % (
options.worker_bind_address, options.worker_socket_address)
else:
print "Starting %s worker process%s. %s." % (
count, count > 1 and 'es'
or '', options.worker_bind_address and
('Listening on "%s"' % options.worker_bind_address) or
('Connecting to "%s"' % options.worker_socket_address))
def main_loop(self):
db_connection = configured_connection()
if options.remote_event_receivers:
from toto.events import EventManager
event_manager = EventManager.instance()
if options.remote_instances:
for address in options.remote_event_receivers.split(','):
event_manager.register_server(address)
init_module = self.__event_init
if init_module:
init_module.invoke(event_manager)
serialization = options.serialization_module and __import__(
options.serialization_module) or pickle
compression = options.compression_module and __import__(
options.compression_module)
worker = TotoWorker(self.__method_module,
options.worker_socket_address, db_connection,
compression, serialization)
if options.startup_function:
startup_path = options.startup_function.rsplit('.')
__import__(startup_path[0]).__dict__[startup_path[1]](
worker=worker, db_connection=db_connection)
worker.start()
def send_worker_command(self, command):
if options.control_socket_address:
socket = zmq.Context().socket(zmq.PUB)
socket.bind(options.control_socket_address)
time.sleep(1)
socket.send_string('command %s' % command)
print "Sent command: %s" % options.command
def run(self):
if options.command:
self.send_worker_command(options.command)
return
super(TotoWorkerService, self).run()
