def result_manager():
results_receiver = create_connection("PULL", RESULT_SENDER_IP, RESULT_SENDER_PORT, "bind")
result_status_sender = create_connection("PUB", RESULT_COLLECTOR_IP, RESULT_COLLECTOR_PORT, "bind")
for task_nbr in range(NUM_JOBS):
result_message = results_receiver.recv_json()
if result_message['quit']:
result_status_sender.send("Quitting!!")
else:
print "Worker %i has done the job. =%i" % (result_message['worker_name'], result_message['result'])
result_status_sender.send("DONE")
time.sleep(5)
def worker(pid):
work_receiver = create_connection("PULL", CONTROLLER_IP, CONTROLLER_PORT, "connect")
results_sender = create_connection("PUSH", RESULT_SENDER_IP, RESULT_SENDER_PORT, "connect")
result_status_receiver = create_connection("SUB", RESULT_COLLECTOR_IP, RESULT_COLLECTOR_PORT, "connect", topic_filter="")
poller = zmq.Poller()
poller.register(work_receiver, zmq.POLLIN)
poller.register(result_status_receiver, zmq.POLLIN)
while True:
socks = dict(poller.poll())
if socks.get(work_receiver) == zmq.POLLIN:
work_message = work_receiver.recv_json()
if work_message.get('quit',0):
print "kill command received from controller..by worker-%s" %pid
results_sender.send_json(work_message)
results_sender.close()
result_status_receiver.close()
work_receiver.close()
break
task_func = get_obj_from_str(work_message['task'])
if task_func:
task_func(work_message['payload'])
product = work_message['num'] * work_message['num']
work_message['worker_name'] = pid
work_message['result'] = product
work_message['finished_at'] = str(datetime.utcnow())
results_sender.send_json(work_message)
if socks.get(result_status_receiver) == zmq.POLLIN:
status_message = result_status_receiver.recv()
if status_message == "DONE":
print("Worker %i received FINSHED, quitting!" % pid)
break
def start_controller(job_type):
controller_send = create_connection("PUSH", \
CONTROLLER_IP, \
CONTROLLER_PORT, \
"bind")
time.sleep(1)
for num in range(NUM_JOBS):
task = { 'task':'send_email',
'payload':{'sender':'',
'recipient':['*****@*****.**']
},
'task_id' : num,
'job_type': job_type,
'num': num,
'quit':False if job_type=='start' else True,
}
controller_send.send_json(task)
time.sleep(1)
log.info('sent task.')
