def do_alarm(alarm_list): global alarm_proxy_host url = 'http://' + alarm_proxy_host url = urljoin(url, '/api/alarm/') my_headers = { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Content-Type': 'application/json;charset=UTF-8', } #my_data = json.dumps(alarm_list) my_data = { 'token': 'testtoken', 'data': [] } my_data['data'] = alarm_list my_data = json.dumps(my_data) slog.info("do_alarm: {0}".format(my_data)) try: #res = requests.post(url, headers = my_headers,data = my_data, timeout = 5) res = mysession.post(url, headers = my_headers,data = my_data, timeout = 5) if res.status_code == 200: if res.json().get('status') == 0: slog.info("send alarm ok, response: {0}".format(res.text)) return True else: slog.warn("send alarm fail, response: {0}".format(res.text)) else: slog.warn('send alarm fail: {0}'.format(res.text)) except Exception as e: slog.warn("exception: {0}".format(e)) return False
def dump_shm_networksize(self): # network_info slog.info("dump network_id to shm") with open(self.network_info_shm_filename_, 'w') as fout: fout.write(json.dumps(self.network_ids_)) fout.close() return
def clear_queue(): global ALARMQ, ALARMQ_HIGH while not ALARMQ.empty(): ALARMQ.get() while not ALARMQ_HIGH.empty(): ALARMQ_HIGH.get() slog.info("clear alarmqueue/alarm_high_queue")
def demo_alarm(self, content): slog.info('demo_alarm begin:{0}'.format(json.dumps(content))) # TODO(user) do something statictis or calculate # dump result to db self.dump_db() return True
def __init__(self, q, queue_key_list, alarm_env='test'): slog.info( "networksize alarmconsumer init. pid:{0} paraent:{1} queue_key:{2}" .format(os.getpid(), os.getppid(), json.dumps(queue_key_list))) self.alarm_env_ = alarm_env # keep all the node_id of some network_id, key is network_id, value is nodes of this network_id # something like {'690000010140ff7f': {'node_info': [{'node_id': xxxx, 'node_ip':127.0.0.1:9000}], 'size':1}} self.network_ids_ = {} # key is public_ip_port, value is {'public_ip_port':'127.0.0.1:9000','rec':[],'zec':[],....,'val':[]} self.node_info_ = {} # key is network_id, value is {network_id: network_id, network_type: (rec/zec/edg/arc/adv/val), network_num:1~10} self.network_id_num_ = {} self.consume_step_ = 30 # store packet_info from /api/alarm self.alarm_queue_ = q self.queue_key_list_ = queue_key_list # eg: topargus_alarm_list:0 for one consumer bind to one or more queue_key # init db obj self.network_info_sql_ = NetworkInfoSql() self.system_alarm_info_sql_ = SystemAlarmInfoSql() self.node_info_sql_ = NodeInfoSql() self.node_info_sql_.delete_db(data={}) self.network_id_num_sql_ = NetworkIdNumSql() self.system_cron_info_sql_ = SystemCronInfoSql() self.network_info_shm_filename_ = '/dev/shm/topargus_network_info' return
def run_watch_stream(filename = './xtop.log'): global ALARMQ,ALARMQ_HIGH clear_queue() offset = 0 while True: time.sleep(1) offset = watchlog(filename, offset) slog.info("grep_log finish, alarmqueue.size = {0} alarmq_high.size = {1}, offset = {2}".format(ALARMQ.qsize(), ALARMQ_HIGH.qsize(), offset))
def dump_gconfig(): global gconfig, gconfig_shm_file with open(gconfig_shm_file, 'w') as fout: fout.write(json.dumps(gconfig)) slog.info('dump gconfig:{0} to file:{1}'.format( json.dumps(gconfig), gconfig_shm_file)) fout.close() return
def dump_db_networksize(self): # network_info slog.info("dump network_id to db") for (k, v) in self.network_ids_.items(): net_data = {'network_id': k, 'network_info': json.dumps(v)} slog.info('dump network_id:{0} size:{1}'.format(k, v.get('size'))) self.network_info_sql_.update_insert_to_db(net_data) return
def run(self): # usually for one consumer , only handle one type slog.info('consume_alarm run') if self.alarm_env_ == 'test': self.consume_alarm_with_notry() else: self.consume_alarm() return
def update_command(): global mycommand while True: time.sleep(60) slog.info('update remote command alive') get_command_from_remote() return
def put_alarmq(alarm_payload): global ALARMQ try: ALARMQ.put(alarm_payload, block=True, timeout =2) slog.info("put send_queue:{0} size:{1}, item:{2}".format(ALARMQ, ALARMQ.qsize(),json.dumps(alarm_payload))) except Exception as e: slog.warn("queue full, drop alarm_payload") return False return True
def put_alarmq_high(alarm_payload): global ALARMQ_HIGH try: ALARMQ_HIGH.put(alarm_payload, block=True, timeout =2) slog.info("put alarm_queue_high:{0} size:{1} item:{2}".format(ALARMQ_HIGH, ALARMQ_HIGH.qsize(),json.dumps(alarm_payload))) except Exception as e: slog.warn("queue full, drop alarm_payload") return False return True
def run(self): # attention, do something init self.load_db_network_id_num() # usually for one consumer , only handle one type slog.info('consume_alarm run') if self.alarm_env_ == 'test': self.consume_alarm_with_notry() else: self.consume_alarm() return
def get_network_ids_exp(self, data): result = self.get_network_ids(data) if data.get('withip') == False: return result result_exp = { 'node_info': [], 'node_size': 0 } slog.debug('get_network_ids_exp') iplocation_update_flag = False iplocation_load_again = False #try: for item in result.get('node_info'): ip = item.get('node_ip').split(':')[0] #''' if ip in self.iplocation_: item['node_country'] = self.iplocation_[ip]['country_name'] else: if not iplocation_update_flag and os.path.exists(self.iplocation_file_): with open(self.iplocation_file_, 'r') as fin: self.iplocation_ = json.loads(fin.read()) iplocation_load_again = True fin.close() slog.info('load iplocation from {0}, size:{1}'.format(self.iplocation_file_, len(self.iplocation_.keys()))) ipinfo = sipinfo.GetIPLocation([ip]) if ipinfo.get(ip): self.iplocation_[ip] = ipinfo.get(ip) item['node_country'] = ipinfo.get(ip).get('country_name') slog.debug('get iplocation of {0} from server'.format(ip)) iplocation_update_flag = True else: item['node_country'] = 'unknow' #''' ''' country_name_list = ['United States', 'China', 'England', 'Afric','France'] tmp_country_name = random.choice(country_name_list) item['node_country'] = tmp_country_name slog.debug('add country {0}'.format(tmp_country_name)) ''' result_exp['node_info'].append(item) if iplocation_update_flag: with open(self.iplocation_file_, 'w') as fout: fout.write(json.dumps(self.iplocation_)) fout.close() result_exp['node_size'] = len(result_exp['node_info']) #except Exception as e: # slog.warn('parse ip goes wrong: {0}'.format(e)) return result_exp
def update_network_ids(self): vs,total = self.query_network_ids({}) if not vs: return False for item in vs: nid = item.get('network_id') ninfo = item.get('network_info') ninfo = json.loads(ninfo) self.network_ids_[nid] = ninfo self.network_ids_['update_timestamp'] = {'update_timestamp': int(time.time() * 1000)} slog.info('read_network_id from db success.') return True
def consume_alarm_with_notry(self): while True: slog.info("begin consume_alarm alarm_queue.size is {0}".format( self.alarm_queue_.qsize(self.queue_key_list_))) alarm_payload_list = self.alarm_queue_.get_queue_exp( self.queue_key_list_, self.consume_step_) # return dict or None for alarm_payload in alarm_payload_list: alarm_type = alarm_payload.get('alarm_type') if alarm_type == 'demo': self.demo_alarm(alarm_payload.get('alarm_content')) else: slog.warn('invalid alarm_type:{0}'.format(alarm_type)) return
def remove_dead_node(self, node_ip): network_ids_bak = copy.deepcopy(self.network_ids_) for k, v in network_ids_bak.items(): for i in range(len(v.get('node_info'))): ni = v.get('node_info')[i] if ni.get('node_ip') == node_ip: del self.network_ids_[k]['node_info'][i] self.network_ids_[k]['size'] -= 1 slog.warn('remove dead node_id:{0} node_ip:{1}'.format( ni.get('node_id'), ni.get('node_ip'))) if len(self.network_ids_[k]['node_info']) == 0: del self.network_ids_[k] for k, v in self.network_ids_.items(): slog.info('network_ids key:{0} size:{1}'.format(k, v.get('size'))) return
def load_db_networksize(self): # TODO(smaug) not use for now return True vs, total = [], 0 vs, total = self.network_info_sql_.query_from_db(data) if not vs: slog.warn('load network_info from db failed') return False for item in vs: self.network_ids_[item.get('network_id')] = json.loads( item.get('network_info')) slog.info( 'load network_info from db ok, network_id:{0} size:{1}'.format( item.get('network_id'), self.network_ids_.get(item.get('network_id')).get('size'))) return True
def __init__(self, q, queue_key_list, alarm_env='test'): slog.info("demo alarmconsumer init. pid:{0} paraent:{1} queue_key:{2}". format(os.getpid(), os.getppid(), json.dumps(queue_key_list))) self.expire_time_ = 20 # 10min, only focus on latest 10 min self.consume_step_ = 20 # get_queue return size for one time self.alarm_env_ = alarm_env # store packet_info from /api/alarm self.alarm_queue_ = q self.queue_key_list_ = queue_key_list # eg: topargus_alarm_list:0 for one consumer bind to one or more queue_key # demo sql db client self.demo_info_sql_ = DemoInfoSql() return
def dump_db_node_info(self, content): # only remove node or add new node if not content: return False node_ip = content.get('node_ip') # ip:port node_id = content.get('node_id') send_timestamp = content.get('send_timestamp') or int( time.time() * 1000) value = copy.deepcopy(self.node_info_.get(node_ip)) if not value: slog.warn('invalid node_id:{0} node_ip:{1}'.format( node_id, node_ip)) return print(value) for k in copy.deepcopy(list(value.keys())): if not value.get(k): value.pop(k) if k in ['rec', 'zec', 'edg', 'arc', 'adv', 'val']: value[k] = json.dumps(value.get(k)) self.node_info_sql_.update_insert_to_db(value) slog.info("dump node_info to db:{0}".format(json.dumps(value))) # upadte system_alarm_info alarm_info = '' root_id = '' priority = PRIORITY_DICT.get('low') node_id_status = content.get('node_id_status') if node_id_status == 'remove': alarm_info = 'remove node_id:{0}'.format(node_id) elif node_id_status == 'dead': root_id = node_id # ffffff alarm_info = 'xtopchain down' priority = PRIORITY_DICT.get('high') else: alarm_info = 'add node_id:{0}'.format(node_id) if not root_id and self.node_info_.get(node_ip): root_id = self.node_info_.get(node_ip).get('root') or '' self.dump_db_system_alarm_info(node_ip, root_id, priority, alarm_info, send_timestamp) return
def dump_db_network_id_num(self, network_id): if network_id.startswith('ffffff'): return if network_id not in self.network_id_num_: self.load_db_network_id_num() if network_id in self.network_id_num_: # already in db return net_type = '' if network_id.startswith('ff0000010000'): net_type = 'rec' elif network_id.startswith('ff0000020000'): net_type = 'zec' elif network_id.startswith('ff00000f0101'): net_type = 'edg' elif network_id.startswith('ff00000e0101'): net_type = 'arc' elif network_id.startswith('ff00000001'): tmp_group_id = int(network_id[-2:], 16) if 0 <= tmp_group_id and tmp_group_id <= 63: # adv group_id: [0, 64) net_type = 'adv' elif 64 <= tmp_group_id and tmp_group_id <= 126: # val group_id: [64, 127) net_type = 'val' else: slog.warn('not support network_id:{0} for map-num'.format( network_id)) return else: slog.warn( 'not support network_id:{0} for map-num'.format(network_id)) return data = {'network_id': network_id, 'network_type': net_type} self.network_id_num_sql_.insert_to_db(data) slog.info('dump network_id_num to db:{0}'.format(json.dumps(data))) self.load_db_network_id_num() return
def consumer_alarm(): global ALARMQ, ALARMQ_HIGH, gconfig alarm_pack_num = gconfig.get('alarm_pack_num') th_name = threading.current_thread().name alarm_list = [] while True: try: slog.info("consumer thread:{0} send_queue:{1} size:{2}".format(th_name, ALARMQ, ALARMQ.qsize())) while not ALARMQ.empty(): alarm_payload = ALARMQ.get() alarm_list.append(alarm_payload) if len(alarm_list) >= alarm_pack_num: slog.info("alarm do_alarm") do_alarm(alarm_list) alarm_list.clear() time.sleep(1) except Exception as e: pass
def consume_alarm(self): while True: slog.info("begin consume_alarm alarm_queue.size is {0}".format( self.alarm_queue_.qsize(self.queue_key_list_))) try: alarm_payload_list = self.alarm_queue_.get_queue_exp( self.queue_key_list_, self.consume_step_) # return dict or None for alarm_payload in alarm_payload_list: alarm_type = alarm_payload.get('alarm_type') if alarm_type == 'networksize': self.networksize_alarm( alarm_payload.get('alarm_content')) elif alarm_type == 'system': self.system_cron_alarm( alarm_payload.get('alarm_content')) else: slog.warn('invalid alarm_type:{0}'.format(alarm_type)) except Exception as e: slog.warn('catch exception:{0}'.format(e)) return
def update_config_from_remote(): global gconfig, alarm_proxy_host, mypublic_ip_port url = 'http://' + alarm_proxy_host url = urljoin(url, '/api/config/') my_headers = { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Content-Type': 'application/json;charset=UTF-8', } config = {} try: res = requests.get(url, headers = my_headers, timeout = 5) if res.status_code == 200: if res.json().get('status') == 0: slog.info("get remote config ok, response: {0}".format(res.text)) config = res.json().get('config') except Exception as e: slog.warn("exception: {0}".format(e)) return False if not config: slog.warn("get remote config fail") return False if dict_cmp(config, gconfig): slog.info("get remote config same as default, no need udpate") return False # TODO(smaug) do something check for config gconfig = copy.deepcopy(config) slog.info('get remote config ok: {0}'.format(json.dumps(gconfig))) return True
def get_command_from_remote(): global alarm_proxy_host, mypublicip, mycommand, mycommand_filter url = 'http://' + alarm_proxy_host url = urljoin(url, '/api/command/') my_headers = { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Content-Type': 'application/json;charset=UTF-8', } config = {} try: #res = requests.get(url, headers = my_headers, timeout = 5) res = mysession.get(url, headers = my_headers, timeout = 5) if res.status_code == 200: if res.json().get('status') == 0: slog.info("get remote command ok, response: {0}".format(res.text)) config = res.json().get('config') command = res.json().get('command') for k,v in command.items(): if k not in mycommand_filter: mycommand.extend(v) mycommand_filter.append(k) ip = res.json().get('ip') if ip != None and ip.find(':') != -1: mypublicip = ip except Exception as e: slog.info("exception: {0}".format(e)) return False slog.info('get remote command ok: {0}'.format(json.dumps(mycommand))) return True
def verify_password(username, password): global user_info print('username:{0}'.format(username)) if username not in user_info: tmp_user_info = myuser.get_user_info() if tmp_user_info: slog.info('update user_info from db:{0}'.format( json.dumps(tmp_user_info))) for item in tmp_user_info: if item.get('username') not in user_info: user_info[item.get('username')] = item.get('password_hash') else: slog.info('update user_info from db failed') if username not in user_info: return False if not check_password_hash(user_info.get(username), password): tmp_user_info = myuser.get_user_info() if not tmp_user_info: return False slog.info('update user_info from db:{0}'.format( json.dumps(tmp_user_info))) for item in tmp_user_info: if item.get('username') != username: continue user_info[username] = item.get('password_hash') return check_password_hash(user_info.get(username), password)
def networksize_alarm(self, content): if not content: return False node_id = content.get('node_id') network_id = node_id[:17] # head 8 * 2 bytes # attention: specially for kroot_id 010000 if network_id.startswith('010000'): network_id = '010000' node_id_status = content.get('node_id_status') if node_id_status == 'remove': if network_id not in self.network_ids_: slog.warn('remove node_id:{0} from nonexistent network_id:{1}'. format(node_id, network_id)) return False for ni in self.network_ids_[network_id]['node_info']: if ni.get('node_id') == node_id: self.network_ids_[network_id]['node_info'].remove(ni) self.network_ids_[network_id]['size'] -= 1 slog.info( 'remove node_id:{0} from network_id:{1}, now size:{2}'. format(node_id, network_id, self.network_ids_[network_id]['size'])) break return True if network_id not in self.network_ids_: network_info = { 'node_info': [{ 'node_id': node_id, 'node_ip': content.get('node_ip') }], 'size': 1, } self.network_ids_[network_id] = network_info slog.info( 'add node_id:{0} to network_id:{1}, new network_id and now size is 1' .format(node_id, network_id)) return True else: for ni in self.network_ids_[network_id]['node_info']: if ni.get('node_id') == node_id: #slog.debug('already exist node_id:{0} in network_id:{1}'.format(node_id, network_id)) return True self.network_ids_[network_id]['node_info'].append({ 'node_id': node_id, 'node_ip': content.get('node_ip') }) self.network_ids_[network_id]['size'] += 1 slog.info( 'add node_id:{0} to network_id:{1}, now size is {2}'.format( node_id, network_id, self.network_ids_[network_id]['size'])) return True return True
def alarm_report(): payload = {} if not request.is_json: payload = json.loads(request.data) else: payload = request.get_json() ret = {'status': ''} status_ret = {0: 'OK', -1: '上报字段不合法,部分可能上传失败', -2: '格式转化出错,请检查字段数或者字段格式等'} if not payload.get('data'): ret = {'status': -2, 'error': status_ret.get(-2)} return jsonify(ret) # TODO(smaug) varify token alarm_ip = request.headers.get('X-Forwarded-For') or request.headers.get( 'X-Real-IP') or request.remote_addr slog.info("recv alarm from ip:{0} size:{1}".format( alarm_ip, len(payload.get('data')))) mq.handle_alarm(payload.get('data')) ret = {'status': 0, 'error': status_ret.get(0)} return jsonify(ret)
def load_db_network_id_num(self): vs, total = [], 0 vs, total = self.network_id_num_sql_.query_from_db(data={}) if not vs: slog.warn('load network_id_num from db failed or empty') return False for item in vs: # just for safety check network_id = item.get('network_id') network_num = item.get('network_num') if network_id in self.network_id_num_: if self.network_id_num_.get(network_id).get( 'network_num') != network_num: slog.warn( 'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}' .format( network_num, self.network_id_num_.get(network_id).get( 'network_num'))) slog.warn( 'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}' .format( network_num, self.network_id_num_.get(network_id).get( 'network_num'))) slog.warn( 'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}' .format( network_num, self.network_id_num_.get(network_id).get( 'network_num'))) sys.exit(-1) self.network_id_num_[item.get('network_id')] = item slog.info('load network_id_num from db success:{0}'.format( json.dumps(self.network_id_num_))) return True
def __init__(self): # init db obj self.packet_info_sql = PacketInfoSql() self.packet_recv_info_sql = PacketRecvInfoSql() self.network_info_sql = NetworkInfoSql() self.packet_drop_rate_sql = DropRateInfoSql() self.node_info_sql_ = NodeInfoSql() self.system_alarm_info_sql_ = SystemAlarmInfoSql() self.network_id_num_sql_ = NetworkIdNumSql() self.system_cron_info_sql_ = SystemCronInfoSql() self.network_ids_lock_ = threading.Lock() self.network_ids_ = {} self.network_id_num_ = {} self.iplocation_ = {} self.iplocation_file_ = '/tmp/.topargus_iplocation' if os.path.exists(self.iplocation_file_): with open(self.iplocation_file_, 'r') as fin: self.iplocation_ = json.loads(fin.read()) fin.close() slog.info('load iplocation from {0}, size:{1}'.format(self.iplocation_file_, len(self.iplocation_.keys()))) return