def result_manager(): results_receiver = create_connection("PULL", RESULT_SENDER_IP, RESULT_SENDER_PORT, "bind") result_status_sender = create_connection("PUB", RESULT_COLLECTOR_IP, RESULT_COLLECTOR_PORT, "bind") for task_nbr in range(NUM_JOBS): result_message = results_receiver.recv_json() if result_message['quit']: result_status_sender.send("Quitting!!") else: print "Worker %i has done the job. =%i" % (result_message['worker_name'], result_message['result']) result_status_sender.send("DONE") time.sleep(5)
def worker(pid): work_receiver = create_connection("PULL", CONTROLLER_IP, CONTROLLER_PORT, "connect") results_sender = create_connection("PUSH", RESULT_SENDER_IP, RESULT_SENDER_PORT, "connect") result_status_receiver = create_connection("SUB", RESULT_COLLECTOR_IP, RESULT_COLLECTOR_PORT, "connect", topic_filter="") poller = zmq.Poller() poller.register(work_receiver, zmq.POLLIN) poller.register(result_status_receiver, zmq.POLLIN) while True: socks = dict(poller.poll()) if socks.get(work_receiver) == zmq.POLLIN: work_message = work_receiver.recv_json() if work_message.get('quit',0): print "kill command received from controller..by worker-%s" %pid results_sender.send_json(work_message) results_sender.close() result_status_receiver.close() work_receiver.close() break task_func = get_obj_from_str(work_message['task']) if task_func: task_func(work_message['payload']) product = work_message['num'] * work_message['num'] work_message['worker_name'] = pid work_message['result'] = product work_message['finished_at'] = str(datetime.utcnow()) results_sender.send_json(work_message) if socks.get(result_status_receiver) == zmq.POLLIN: status_message = result_status_receiver.recv() if status_message == "DONE": print("Worker %i received FINSHED, quitting!" % pid) break
def start_controller(job_type): controller_send = create_connection("PUSH", \ CONTROLLER_IP, \ CONTROLLER_PORT, \ "bind") time.sleep(1) for num in range(NUM_JOBS): task = { 'task':'send_email', 'payload':{'sender':'', 'recipient':['*****@*****.**'] }, 'task_id' : num, 'job_type': job_type, 'num': num, 'quit':False if job_type=='start' else True, } controller_send.send_json(task) time.sleep(1) log.info('sent task.')