def init_app(self): # init parent class, make oslo take effect super(LawrenceApplication, self).init_app() # init log logging.setup("lawrence instance.") LOG.info("server instance is runing.") # init redis cluster self.redis = RedisOperation() self.redis.connect() # init pool(max value is 50) self.pool = multiprocessing.Pool(4, init_signal_handler) # init socket long connect self.sock_obj = QSocketOperation() host_list = CONF.NODES.nodes proxy_port = CONF.PROXY.proxy_port timeout = 4 self.sock_obj.connect(host_list, proxy_port, timeout)
class LawrenceApplication(QApplication): name = "lawrence" version = "v1.0" _last_update = 0 def init_app(self): # init parent class, make oslo take effect super(LawrenceApplication, self).init_app() # init log logging.setup("lawrence instance.") LOG.info("server instance is runing.") # init redis cluster self.redis = RedisOperation() self.redis.connect() # init pool(max value is 50) self.pool = multiprocessing.Pool(4, init_signal_handler) # init socket long connect self.sock_obj = QSocketOperation() host_list = CONF.NODES.nodes proxy_port = CONF.PROXY.proxy_port timeout = 4 self.sock_obj.connect(host_list, proxy_port, timeout) def run(self): while True: # heartbeat_check heartbeat = struct.pack("!II", HEARTBEAT, 0) send_status = self.sock_obj.sendmsg(heartbeat) if not send_status: LOG.warn("____proxy proxy master is badding, need alarm.") # request_task self._get_taskinfo() head_data = self.sock_obj.recvmsg(HEADER_BYTES) if head_data: head_msg = struct.unpack('!II', head_data) command_id = head_msg[0] if command_id == ACK_HEART: pass elif command_id == ACK_NORMAL: package_len = head_msg[1] bodylen = package_len - HEADER_BYTES body = self.sock_obj.recvmsg(bodylen) LOG.info("receive body for task data is: %s" % body) if not body: time.sleep(2) continue # do task task_list = json.loads(body) self._dispatch_task(task_list) time.sleep(2) def _get_taskinfo(self): cur_time = time.time() timestamp = long(cur_time) timedelta = timestamp - self._last_update if timedelta < 50: return curSecond = long(time.strftime("%S", time.localtime(cur_time))) if curSecond < 30: return self._last_update = timestamp request_data = struct.pack("!II", NORMALMSG, 0) send_status = self.sock_obj.sendmsg(request_data) if not send_status: LOG.info("request task data is failed.........") def _dispatch_task(self, task_list): try: for item in task_list: group = item.get("group") is_exist = self.redis.exists(group) if is_exist: self.redis.delete(group) LOG.warn("find unfinished task, group is (%s)" % group) continue self.redis.set_redis_cache(group, item) # Note(apply_async‘s args must be a single function.) self.pool.apply_async(collector, (item,)) LOG.info("finish all tasks assigned.") except KeyboardInterrupt: self.pool.terminate() self.pool.close()