def put_data(self, data_seq): '''获取数据put到salve的队列中''' try: for data in data_seq: self.queue.put_nowait(data) self.client.send(Message('done', None, self.client_id)) self.state = STATE_DONE except Full: self.logger.warn("client %s queue has been full!" % self.client_id) self.client.send(Message('done', None, self.client_id)) except Exception as e: self.logger.error("Slave put data error", exc_info=True) return
def on_receive_master_data(data_seq): '''func doc''' if self.state == STATE_DONE: self.client.send(Message('done', data_seq, self.client_id)) else: self.logger.info("start put data into queue!") self.put_data(data_seq)
def stop(self): '''停掉所有slave并重置master''' # self.state = STATE_STOPPED self.stats.clear() self._reset_master() for _ in self.clients: self.server.send(Message("stop", None, None))
def start_distribute(self): '''分发数据入口方法,供web接口调用''' self.state = STATE_RUNNING for _ in six.itervalues(self.clients): self.server.send(Message('start', None, None)) self.logger.info("Master start distributing data!") self.greenlet.spawn(self.distributer).join(timeout=30) return True
def __init__(self): '''func doc''' super(SlaveRunner, self).__init__() self.logger = get_logger('data_runner.slave') self.client_id = socket.gethostname() + "_" + md5( str(time.time() + random.randint(0, 10000)).encode('utf-8')).hexdigest() self.client = rpc.Client(self.master_host, self.master_bind_port) self.greenlet.spawn(self.worker) self.client.send(Message('connected', None, self.client_id)) self.greenlet.spawn(self.reporter) def on_report_to_master(client_id, data): '''func doc''' data["state"] = self.state data["queue_size"] = self.queue_size data["data_count"] = self.current_queue_size data["cpu_percent"] = self.cpu_info data["mem_percent"] = self.mem_info data["loadavg"] = self.sys_loadavg events.report_to_master += on_report_to_master def on_make_slave_queue(qsize): '''func doc''' self.state = STATE_READY self.queue_size = qsize self.queue = Queue(self.queue_size) self.client.send(Message('ready', qsize, self.client_id)) events.make_slave_queue += on_make_slave_queue def on_receive_master_data(data_seq): '''func doc''' if self.state == STATE_DONE: self.client.send(Message('done', data_seq, self.client_id)) else: self.logger.info("start put data into queue!") self.put_data(data_seq) events.receive_master_data += on_receive_master_data def on_reset_slave(): '''func doc''' self.state = STATE_INIT self.queue = None self.queue_size = 0 self.client.send(Message('client_reset', None, self.client_id)) events.reset_slave += on_reset_slave def on_slave_stopping(): self.client.send(Message('stopped', None, self.client_id)) self.stop() events.slave_stopping += on_slave_stopping
def reporter(self): '''slave数据上报线程''' while self.state != STATE_STOPPED: data = {} events.report_to_master.fire(client_id=self.client_id, data=data) try: self.client.send(Message("stats", data, self.client_id)) except: self.logger.error( "Connection lost to master server. Aborting...", exc_info=True) break gevent.sleep(SLAVE_REPORT_INTERVAL) else: self.logger.info("Slave reporter is stopped!")
def reset(self): '''重置各节点状态''' self.state = STATE_INIT self.queue = None self.queue_size = 0 if hasattr(self, "clients"): self.init_info = MainNode(self.master_host) events.runner_data_refresh.fire(self.init_info) for client in six.itervalues(self.clients): client.qsize = 0 self.server.send(Message('reset', None, None)) elif hasattr(self, "client_id"): pass else: self.init_info = MainNode(id=self.id, role="local") events.runner_data_refresh.fire(self.init_info)
def distributer(self): '''按照各client的qsize分发数据''' try: for client in six.itervalues(self.clients): temp_data = [] for _ in xrange(client.qsize): data = self.queue.get(timeout=2) temp_data.append(data) self.server.send(Message('data', temp_data, None)) self.logger.info("send data to slaves:{0}".format( temp_data[0])) except Empty: raise except: self.logger.error("Distribute error", exc_info=True) finally: self.state = STATE_DONE self.logger.info("All data of master have been distributed!")
def worker(self): '''slave消息监听线程''' self.logger.info("Slave worker start!") while self.state != STATE_STOPPED: msg = self.client.recv() if msg.type == 'prepare': events.make_slave_queue.fire(msg.data['queue_size']) elif msg.type == 'start': self.state = STATE_RUNNING self.client.send(Message('client_start', None, self.client_id)) elif msg.type == 'data': self.logger.info("receive data from master:{0}".format( msg.data[0])) events.receive_master_data.fire(msg.data) self.logger.info("handle receive data done!") elif msg.type == 'reset': events.reset_slave.fire() elif msg.type == 'stop': events.slave_stopping.fire() else: self.logger.info("Slave worker stopped!")
def prepare_distribute(self): '''通知slave创建对应长度的数据队列,为接收数据做准备''' if self.state != STATE_READY: self.logger.warn( "Master data is not ready!wait 2 second then continue") gevent.sleep(2) self.prepare_distribute() num_slaves = len(self.clients.connected) if not num_slaves: self.logger.warn("no slave servers connected!") return slave_queue_size = self.queue_size // num_slaves or 1 remaining = self.queue_size % num_slaves for client in six.itervalues(self.clients): data = {"queue_size": slave_queue_size, "stop_timeout": None} if remaining > 0: data['queue_size'] += 1 remaining -= 1 client.qsize = data['queue_size'] self.logger.info("%s queue size: %s" % (client.id, client.qsize)) self.server.send(Message("prepare", data, None))
def on_slave_stopping(): self.client.send(Message('stopped', None, self.client_id)) self.stop()
def on_reset_slave(): '''func doc''' self.state = STATE_INIT self.queue = None self.queue_size = 0 self.client.send(Message('client_reset', None, self.client_id))
def on_make_slave_queue(qsize): '''func doc''' self.state = STATE_READY self.queue_size = qsize self.queue = Queue(self.queue_size) self.client.send(Message('ready', qsize, self.client_id))