def put_data(self, data_seq):
'''获取数据put到salve的队列中'''
try:
for data in data_seq:
self.queue.put_nowait(data)
self.client.send(Message('done', None, self.client_id))
self.state = STATE_DONE
except Full:
self.logger.warn("client %s queue has been full!" % self.client_id)
self.client.send(Message('done', None, self.client_id))
except Exception as e:
self.logger.error("Slave put data error", exc_info=True)
return
def on_receive_master_data(data_seq):
'''func doc'''
if self.state == STATE_DONE:
self.client.send(Message('done', data_seq, self.client_id))
else:
self.logger.info("start put data into queue!")
self.put_data(data_seq)
def stop(self):
'''停掉所有slave并重置master'''
# self.state = STATE_STOPPED
self.stats.clear()
self._reset_master()
for _ in self.clients:
self.server.send(Message("stop", None, None))
def start_distribute(self):
'''分发数据入口方法，供web接口调用'''
self.state = STATE_RUNNING
for _ in six.itervalues(self.clients):
self.server.send(Message('start', None, None))
self.logger.info("Master start distributing data!")
self.greenlet.spawn(self.distributer).join(timeout=30)
return True
def __init__(self):
'''func doc'''
super(SlaveRunner, self).__init__()
self.logger = get_logger('data_runner.slave')
self.client_id = socket.gethostname() + "_" + md5(
str(time.time() +
random.randint(0, 10000)).encode('utf-8')).hexdigest()
self.client = rpc.Client(self.master_host, self.master_bind_port)
self.greenlet.spawn(self.worker)
self.client.send(Message('connected', None, self.client_id))
self.greenlet.spawn(self.reporter)
def on_report_to_master(client_id, data):
'''func doc'''
data["state"] = self.state
data["queue_size"] = self.queue_size
data["data_count"] = self.current_queue_size
data["cpu_percent"] = self.cpu_info
data["mem_percent"] = self.mem_info
data["loadavg"] = self.sys_loadavg
events.report_to_master += on_report_to_master
def on_make_slave_queue(qsize):
'''func doc'''
self.state = STATE_READY
self.queue_size = qsize
self.queue = Queue(self.queue_size)
self.client.send(Message('ready', qsize, self.client_id))
events.make_slave_queue += on_make_slave_queue
def on_receive_master_data(data_seq):
'''func doc'''
if self.state == STATE_DONE:
self.client.send(Message('done', data_seq, self.client_id))
else:
self.logger.info("start put data into queue!")
self.put_data(data_seq)
events.receive_master_data += on_receive_master_data
def on_reset_slave():
'''func doc'''
self.state = STATE_INIT
self.queue = None
self.queue_size = 0
self.client.send(Message('client_reset', None, self.client_id))
events.reset_slave += on_reset_slave
def on_slave_stopping():
self.client.send(Message('stopped', None, self.client_id))
self.stop()
events.slave_stopping += on_slave_stopping
def reporter(self):
'''slave数据上报线程'''
while self.state != STATE_STOPPED:
data = {}
events.report_to_master.fire(client_id=self.client_id, data=data)
try:
self.client.send(Message("stats", data, self.client_id))
except:
self.logger.error(
"Connection lost to master server. Aborting...",
exc_info=True)
break
gevent.sleep(SLAVE_REPORT_INTERVAL)
else:
self.logger.info("Slave reporter is stopped!")
def reset(self):
'''重置各节点状态'''
self.state = STATE_INIT
self.queue = None
self.queue_size = 0
if hasattr(self, "clients"):
self.init_info = MainNode(self.master_host)
events.runner_data_refresh.fire(self.init_info)
for client in six.itervalues(self.clients):
client.qsize = 0
self.server.send(Message('reset', None, None))
elif hasattr(self, "client_id"):
pass
else:
self.init_info = MainNode(id=self.id, role="local")
events.runner_data_refresh.fire(self.init_info)
def distributer(self):
'''按照各client的qsize分发数据'''
try:
for client in six.itervalues(self.clients):
temp_data = []
for _ in xrange(client.qsize):
data = self.queue.get(timeout=2)
temp_data.append(data)
self.server.send(Message('data', temp_data, None))
self.logger.info("send data to slaves:{0}".format(
temp_data[0]))
except Empty:
raise
except:
self.logger.error("Distribute error", exc_info=True)
finally:
self.state = STATE_DONE
self.logger.info("All data of master have been distributed!")
def worker(self):
'''slave消息监听线程'''
self.logger.info("Slave worker start!")
while self.state != STATE_STOPPED:
msg = self.client.recv()
if msg.type == 'prepare':
events.make_slave_queue.fire(msg.data['queue_size'])
elif msg.type == 'start':
self.state = STATE_RUNNING
self.client.send(Message('client_start', None, self.client_id))
elif msg.type == 'data':
self.logger.info("receive data from master:{0}".format(
msg.data[0]))
events.receive_master_data.fire(msg.data)
self.logger.info("handle receive data done!")
elif msg.type == 'reset':
events.reset_slave.fire()
elif msg.type == 'stop':
events.slave_stopping.fire()
else:
self.logger.info("Slave worker stopped!")
def prepare_distribute(self):
'''通知slave创建对应长度的数据队列，为接收数据做准备'''
if self.state != STATE_READY:
self.logger.warn(
"Master data is not ready!wait 2 second then continue")
gevent.sleep(2)
self.prepare_distribute()
num_slaves = len(self.clients.connected)
if not num_slaves:
self.logger.warn("no slave servers connected!")
return
slave_queue_size = self.queue_size // num_slaves or 1
remaining = self.queue_size % num_slaves
for client in six.itervalues(self.clients):
data = {"queue_size": slave_queue_size, "stop_timeout": None}
if remaining > 0:
data['queue_size'] += 1
remaining -= 1
client.qsize = data['queue_size']
self.logger.info("%s queue size: %s" % (client.id, client.qsize))
self.server.send(Message("prepare", data, None))
def on_slave_stopping():
self.client.send(Message('stopped', None, self.client_id))
self.stop()
def on_reset_slave():
'''func doc'''
self.state = STATE_INIT
self.queue = None
self.queue_size = 0
self.client.send(Message('client_reset', None, self.client_id))
def on_make_slave_queue(qsize):
'''func doc'''
self.state = STATE_READY
self.queue_size = qsize
self.queue = Queue(self.queue_size)
self.client.send(Message('ready', qsize, self.client_id))