def test_quitsOnCommand(self):
use_threading = True
callback = lambda x: x * 100
context = zmq.Context()
# bind the command_sender socket
command_sender = context.socket(zmq.PUB)
command_sender.bind("inproc://commands")
# launch the cluster manager
# request_sender and result_receiver bind
# command_receiver connects
request_sender = Connect("inproc://requests", bind=True)
result_receiver = Connect("inproc://results", bind=True)
command_receiver = Connect(
"inproc://commands",
options={zmq.SUBSCRIBE: ""}) # subscribe to all command messages
manager = ClusterManager(context,
request_sender,
result_receiver,
command_sender,
command_receiver,
use_threading=use_threading,
receiver_timeout=RECEIVER_TIMEOUT)
manager.Setup()
time.sleep(1)
# launch the worker
# all sockets connect
request_receiver = Connect("inproc://requests")
result_sender = Connect("inproc://results")
worker = threading.Thread(target=WorkerThreadTarget,
args=(context, request_receiver,
result_sender, callback,
command_receiver))
worker.daemon = True
worker.start()
# tear down the worker
Worker.SendKillCommand(context, command_sender)
worker.join(100)
self.assertFalse(worker.is_alive())
# tear down the cluster manager
manager.Shutdown()
self.assertFalse(manager.sink.is_alive())
def __init__(self, config=None, manager=None, chunksize=None):
"""Create a new object.
config -- (ClusterConfig) a cluster configuration. Used to create a
ClusterManager if none is given.
manager -- (ClusterManager) the manager with which to process requests
"""
if manager == None:
assert config != None, "Must specify ClusterConfig or ClusterManager"
context = zmq.Context()
manager = ClusterManager(context, config.request_sender,
config.result_receiver,
config.command_sender,
config.command_receiver)
if chunksize == None:
chunksize = 16 # by default, send enough work to occupy a 16-core machine
self.manager = manager
self.chunksize = chunksize
self.cluster_stats = {}
def test_map(self):
use_threading = True
callback = lambda x: x * 100
context = zmq.Context()
# bind the command_sender socket
command_sender = Connect(url="ipc://commands.ipc", bind=True)
# launch the cluster manager
# request_sender and result_receiver bind
# command_receiver connects
request_sender = Connect("ipc://requests.ipc", bind=True)
result_receiver = Connect("ipc://results.ipc", bind=True)
command_receiver = Connect(
"ipc://commands.ipc",
options={zmq.SUBSCRIBE: ""}) # subscribe to all command messages
manager = ClusterManager(context,
request_sender,
result_receiver,
command_sender,
command_receiver,
use_threading=use_threading,
receiver_timeout=RECEIVER_TIMEOUT)
manager.Setup()
time.sleep(1)
# launch the worker
# all sockets connect
request_receiver = Connect("ipc://requests.ipc")
result_sender = Connect("ipc://results.ipc")
worker = multiprocessing.Process(target=WorkerProcessTarget,
args=(request_receiver, result_sender,
callback, command_receiver))
worker.daemon = True
worker.start()
xs = range(10)
put_size = manager.PutMany(xs)
self.assertEqual(len(xs), put_size)
expected_ys = map(callback, xs)
actual_ys = sorted(manager.GetMany(len(xs)))
self.assertEqual(expected_ys, actual_ys)
# tear down the worker
Worker.SendKillCommand(context, command_sender)
worker.join(100)
self.assertFalse(worker.is_alive())
# tear down the cluster manager
manager.Shutdown()
self.assertFalse(manager.sink.is_alive())