def do_alarm(alarm_list):
global alarm_proxy_host
url = 'http://' + alarm_proxy_host
url = urljoin(url, '/api/alarm/')
my_headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
}
#my_data = json.dumps(alarm_list)
my_data = {
'token': 'testtoken',
'data': []
}
my_data['data'] = alarm_list
my_data = json.dumps(my_data)
slog.info("do_alarm: {0}".format(my_data))
try:
#res = requests.post(url, headers = my_headers,data = my_data, timeout = 5)
res = mysession.post(url, headers = my_headers,data = my_data, timeout = 5)
if res.status_code == 200:
if res.json().get('status') == 0:
slog.info("send alarm ok, response: {0}".format(res.text))
return True
else:
slog.warn("send alarm fail, response: {0}".format(res.text))
else:
slog.warn('send alarm fail: {0}'.format(res.text))
except Exception as e:
slog.warn("exception: {0}".format(e))
return False
def dump_shm_networksize(self):
# network_info
slog.info("dump network_id to shm")
with open(self.network_info_shm_filename_, 'w') as fout:
fout.write(json.dumps(self.network_ids_))
fout.close()
return
def clear_queue():
global ALARMQ, ALARMQ_HIGH
while not ALARMQ.empty():
ALARMQ.get()
while not ALARMQ_HIGH.empty():
ALARMQ_HIGH.get()
slog.info("clear alarmqueue/alarm_high_queue")
def demo_alarm(self, content):
slog.info('demo_alarm begin:{0}'.format(json.dumps(content)))
# TODO(user) do something statictis or calculate
# dump result to db
self.dump_db()
return True
def __init__(self, q, queue_key_list, alarm_env='test'):
slog.info(
"networksize alarmconsumer init. pid:{0} paraent:{1} queue_key:{2}"
.format(os.getpid(), os.getppid(), json.dumps(queue_key_list)))
self.alarm_env_ = alarm_env
# keep all the node_id of some network_id, key is network_id, value is nodes of this network_id
# something like {'690000010140ff7f': {'node_info': [{'node_id': xxxx, 'node_ip':127.0.0.1:9000}], 'size':1}}
self.network_ids_ = {}
# key is public_ip_port, value is {'public_ip_port':'127.0.0.1:9000','rec':[],'zec':[],....,'val':[]}
self.node_info_ = {}
# key is network_id, value is {network_id: network_id, network_type: (rec/zec/edg/arc/adv/val), network_num:1~10}
self.network_id_num_ = {}
self.consume_step_ = 30
# store packet_info from /api/alarm
self.alarm_queue_ = q
self.queue_key_list_ = queue_key_list # eg: topargus_alarm_list:0 for one consumer bind to one or more queue_key
# init db obj
self.network_info_sql_ = NetworkInfoSql()
self.system_alarm_info_sql_ = SystemAlarmInfoSql()
self.node_info_sql_ = NodeInfoSql()
self.node_info_sql_.delete_db(data={})
self.network_id_num_sql_ = NetworkIdNumSql()
self.system_cron_info_sql_ = SystemCronInfoSql()
self.network_info_shm_filename_ = '/dev/shm/topargus_network_info'
return
def run_watch_stream(filename = './xtop.log'):
global ALARMQ,ALARMQ_HIGH
clear_queue()
offset = 0
while True:
time.sleep(1)
offset = watchlog(filename, offset)
slog.info("grep_log finish, alarmqueue.size = {0} alarmq_high.size = {1}, offset = {2}".format(ALARMQ.qsize(), ALARMQ_HIGH.qsize(), offset))
def dump_gconfig():
global gconfig, gconfig_shm_file
with open(gconfig_shm_file, 'w') as fout:
fout.write(json.dumps(gconfig))
slog.info('dump gconfig:{0} to file:{1}'.format(
json.dumps(gconfig), gconfig_shm_file))
fout.close()
return
def dump_db_networksize(self):
# network_info
slog.info("dump network_id to db")
for (k, v) in self.network_ids_.items():
net_data = {'network_id': k, 'network_info': json.dumps(v)}
slog.info('dump network_id:{0} size:{1}'.format(k, v.get('size')))
self.network_info_sql_.update_insert_to_db(net_data)
return
def run(self):
# usually for one consumer , only handle one type
slog.info('consume_alarm run')
if self.alarm_env_ == 'test':
self.consume_alarm_with_notry()
else:
self.consume_alarm()
return
def update_command():
global mycommand
while True:
time.sleep(60)
slog.info('update remote command alive')
get_command_from_remote()
return
def put_alarmq(alarm_payload):
global ALARMQ
try:
ALARMQ.put(alarm_payload, block=True, timeout =2)
slog.info("put send_queue:{0} size:{1}, item:{2}".format(ALARMQ, ALARMQ.qsize(),json.dumps(alarm_payload)))
except Exception as e:
slog.warn("queue full, drop alarm_payload")
return False
return True
def put_alarmq_high(alarm_payload):
global ALARMQ_HIGH
try:
ALARMQ_HIGH.put(alarm_payload, block=True, timeout =2)
slog.info("put alarm_queue_high:{0} size:{1} item:{2}".format(ALARMQ_HIGH, ALARMQ_HIGH.qsize(),json.dumps(alarm_payload)))
except Exception as e:
slog.warn("queue full, drop alarm_payload")
return False
return True
def run(self):
# attention, do something init
self.load_db_network_id_num()
# usually for one consumer , only handle one type
slog.info('consume_alarm run')
if self.alarm_env_ == 'test':
self.consume_alarm_with_notry()
else:
self.consume_alarm()
return
def get_network_ids_exp(self, data):
result = self.get_network_ids(data)
if data.get('withip') == False:
return result
result_exp = {
'node_info': [],
'node_size': 0
}
slog.debug('get_network_ids_exp')
iplocation_update_flag = False
iplocation_load_again = False
#try:
for item in result.get('node_info'):
ip = item.get('node_ip').split(':')[0]
#'''
if ip in self.iplocation_:
item['node_country'] = self.iplocation_[ip]['country_name']
else:
if not iplocation_update_flag and os.path.exists(self.iplocation_file_):
with open(self.iplocation_file_, 'r') as fin:
self.iplocation_ = json.loads(fin.read())
iplocation_load_again = True
fin.close()
slog.info('load iplocation from {0}, size:{1}'.format(self.iplocation_file_, len(self.iplocation_.keys())))
ipinfo = sipinfo.GetIPLocation([ip])
if ipinfo.get(ip):
self.iplocation_[ip] = ipinfo.get(ip)
item['node_country'] = ipinfo.get(ip).get('country_name')
slog.debug('get iplocation of {0} from server'.format(ip))
iplocation_update_flag = True
else:
item['node_country'] = 'unknow'
#'''
'''
country_name_list = ['United States', 'China', 'England', 'Afric','France']
tmp_country_name = random.choice(country_name_list)
item['node_country'] = tmp_country_name
slog.debug('add country {0}'.format(tmp_country_name))
'''
result_exp['node_info'].append(item)
if iplocation_update_flag:
with open(self.iplocation_file_, 'w') as fout:
fout.write(json.dumps(self.iplocation_))
fout.close()
result_exp['node_size'] = len(result_exp['node_info'])
#except Exception as e:
# slog.warn('parse ip goes wrong: {0}'.format(e))
return result_exp
def update_network_ids(self):
vs,total = self.query_network_ids({})
if not vs:
return False
for item in vs:
nid = item.get('network_id')
ninfo = item.get('network_info')
ninfo = json.loads(ninfo)
self.network_ids_[nid] = ninfo
self.network_ids_['update_timestamp'] = {'update_timestamp': int(time.time() * 1000)}
slog.info('read_network_id from db success.')
return True
def consume_alarm_with_notry(self):
while True:
slog.info("begin consume_alarm alarm_queue.size is {0}".format(
self.alarm_queue_.qsize(self.queue_key_list_)))
alarm_payload_list = self.alarm_queue_.get_queue_exp(
self.queue_key_list_,
self.consume_step_) # return dict or None
for alarm_payload in alarm_payload_list:
alarm_type = alarm_payload.get('alarm_type')
if alarm_type == 'demo':
self.demo_alarm(alarm_payload.get('alarm_content'))
else:
slog.warn('invalid alarm_type:{0}'.format(alarm_type))
return
def remove_dead_node(self, node_ip):
network_ids_bak = copy.deepcopy(self.network_ids_)
for k, v in network_ids_bak.items():
for i in range(len(v.get('node_info'))):
ni = v.get('node_info')[i]
if ni.get('node_ip') == node_ip:
del self.network_ids_[k]['node_info'][i]
self.network_ids_[k]['size'] -= 1
slog.warn('remove dead node_id:{0} node_ip:{1}'.format(
ni.get('node_id'), ni.get('node_ip')))
if len(self.network_ids_[k]['node_info']) == 0:
del self.network_ids_[k]
for k, v in self.network_ids_.items():
slog.info('network_ids key:{0} size:{1}'.format(k, v.get('size')))
return
def load_db_networksize(self):
# TODO(smaug) not use for now
return True
vs, total = [], 0
vs, total = self.network_info_sql_.query_from_db(data)
if not vs:
slog.warn('load network_info from db failed')
return False
for item in vs:
self.network_ids_[item.get('network_id')] = json.loads(
item.get('network_info'))
slog.info(
'load network_info from db ok, network_id:{0} size:{1}'.format(
item.get('network_id'),
self.network_ids_.get(item.get('network_id')).get('size')))
return True
def __init__(self, q, queue_key_list, alarm_env='test'):
slog.info("demo alarmconsumer init. pid:{0} paraent:{1} queue_key:{2}".
format(os.getpid(), os.getppid(),
json.dumps(queue_key_list)))
self.expire_time_ = 20 # 10min, only focus on latest 10 min
self.consume_step_ = 20 # get_queue return size for one time
self.alarm_env_ = alarm_env
# store packet_info from /api/alarm
self.alarm_queue_ = q
self.queue_key_list_ = queue_key_list # eg: topargus_alarm_list:0 for one consumer bind to one or more queue_key
# demo sql db client
self.demo_info_sql_ = DemoInfoSql()
return
def dump_db_node_info(self, content):
# only remove node or add new node
if not content:
return False
node_ip = content.get('node_ip') # ip:port
node_id = content.get('node_id')
send_timestamp = content.get('send_timestamp') or int(
time.time() * 1000)
value = copy.deepcopy(self.node_info_.get(node_ip))
if not value:
slog.warn('invalid node_id:{0} node_ip:{1}'.format(
node_id, node_ip))
return
print(value)
for k in copy.deepcopy(list(value.keys())):
if not value.get(k):
value.pop(k)
if k in ['rec', 'zec', 'edg', 'arc', 'adv', 'val']:
value[k] = json.dumps(value.get(k))
self.node_info_sql_.update_insert_to_db(value)
slog.info("dump node_info to db:{0}".format(json.dumps(value)))
# upadte system_alarm_info
alarm_info = ''
root_id = ''
priority = PRIORITY_DICT.get('low')
node_id_status = content.get('node_id_status')
if node_id_status == 'remove':
alarm_info = 'remove node_id:{0}'.format(node_id)
elif node_id_status == 'dead':
root_id = node_id # ffffff
alarm_info = 'xtopchain down'
priority = PRIORITY_DICT.get('high')
else:
alarm_info = 'add node_id:{0}'.format(node_id)
if not root_id and self.node_info_.get(node_ip):
root_id = self.node_info_.get(node_ip).get('root') or ''
self.dump_db_system_alarm_info(node_ip, root_id, priority, alarm_info,
send_timestamp)
return
def dump_db_network_id_num(self, network_id):
if network_id.startswith('ffffff'):
return
if network_id not in self.network_id_num_:
self.load_db_network_id_num()
if network_id in self.network_id_num_:
# already in db
return
net_type = ''
if network_id.startswith('ff0000010000'):
net_type = 'rec'
elif network_id.startswith('ff0000020000'):
net_type = 'zec'
elif network_id.startswith('ff00000f0101'):
net_type = 'edg'
elif network_id.startswith('ff00000e0101'):
net_type = 'arc'
elif network_id.startswith('ff00000001'):
tmp_group_id = int(network_id[-2:], 16)
if 0 <= tmp_group_id and tmp_group_id <= 63:
# adv group_id: [0, 64)
net_type = 'adv'
elif 64 <= tmp_group_id and tmp_group_id <= 126:
# val group_id: [64, 127)
net_type = 'val'
else:
slog.warn('not support network_id:{0} for map-num'.format(
network_id))
return
else:
slog.warn(
'not support network_id:{0} for map-num'.format(network_id))
return
data = {'network_id': network_id, 'network_type': net_type}
self.network_id_num_sql_.insert_to_db(data)
slog.info('dump network_id_num to db:{0}'.format(json.dumps(data)))
self.load_db_network_id_num()
return
def consumer_alarm():
global ALARMQ, ALARMQ_HIGH, gconfig
alarm_pack_num = gconfig.get('alarm_pack_num')
th_name = threading.current_thread().name
alarm_list = []
while True:
try:
slog.info("consumer thread:{0} send_queue:{1} size:{2}".format(th_name, ALARMQ, ALARMQ.qsize()))
while not ALARMQ.empty():
alarm_payload = ALARMQ.get()
alarm_list.append(alarm_payload)
if len(alarm_list) >= alarm_pack_num:
slog.info("alarm do_alarm")
do_alarm(alarm_list)
alarm_list.clear()
time.sleep(1)
except Exception as e:
pass
def consume_alarm(self):
while True:
slog.info("begin consume_alarm alarm_queue.size is {0}".format(
self.alarm_queue_.qsize(self.queue_key_list_)))
try:
alarm_payload_list = self.alarm_queue_.get_queue_exp(
self.queue_key_list_,
self.consume_step_) # return dict or None
for alarm_payload in alarm_payload_list:
alarm_type = alarm_payload.get('alarm_type')
if alarm_type == 'networksize':
self.networksize_alarm(
alarm_payload.get('alarm_content'))
elif alarm_type == 'system':
self.system_cron_alarm(
alarm_payload.get('alarm_content'))
else:
slog.warn('invalid alarm_type:{0}'.format(alarm_type))
except Exception as e:
slog.warn('catch exception:{0}'.format(e))
return
def update_config_from_remote():
global gconfig, alarm_proxy_host, mypublic_ip_port
url = 'http://' + alarm_proxy_host
url = urljoin(url, '/api/config/')
my_headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
}
config = {}
try:
res = requests.get(url, headers = my_headers, timeout = 5)
if res.status_code == 200:
if res.json().get('status') == 0:
slog.info("get remote config ok, response: {0}".format(res.text))
config = res.json().get('config')
except Exception as e:
slog.warn("exception: {0}".format(e))
return False
if not config:
slog.warn("get remote config fail")
return False
if dict_cmp(config, gconfig):
slog.info("get remote config same as default, no need udpate")
return False
# TODO(smaug) do something check for config
gconfig = copy.deepcopy(config)
slog.info('get remote config ok: {0}'.format(json.dumps(gconfig)))
return True
def get_command_from_remote():
global alarm_proxy_host, mypublicip, mycommand, mycommand_filter
url = 'http://' + alarm_proxy_host
url = urljoin(url, '/api/command/')
my_headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
}
config = {}
try:
#res = requests.get(url, headers = my_headers, timeout = 5)
res = mysession.get(url, headers = my_headers, timeout = 5)
if res.status_code == 200:
if res.json().get('status') == 0:
slog.info("get remote command ok, response: {0}".format(res.text))
config = res.json().get('config')
command = res.json().get('command')
for k,v in command.items():
if k not in mycommand_filter:
mycommand.extend(v)
mycommand_filter.append(k)
ip = res.json().get('ip')
if ip != None and ip.find(':') != -1:
mypublicip = ip
except Exception as e:
slog.info("exception: {0}".format(e))
return False
slog.info('get remote command ok: {0}'.format(json.dumps(mycommand)))
return True
def verify_password(username, password):
global user_info
print('username:{0}'.format(username))
if username not in user_info:
tmp_user_info = myuser.get_user_info()
if tmp_user_info:
slog.info('update user_info from db:{0}'.format(
json.dumps(tmp_user_info)))
for item in tmp_user_info:
if item.get('username') not in user_info:
user_info[item.get('username')] = item.get('password_hash')
else:
slog.info('update user_info from db failed')
if username not in user_info:
return False
if not check_password_hash(user_info.get(username), password):
tmp_user_info = myuser.get_user_info()
if not tmp_user_info:
return False
slog.info('update user_info from db:{0}'.format(
json.dumps(tmp_user_info)))
for item in tmp_user_info:
if item.get('username') != username:
continue
user_info[username] = item.get('password_hash')
return check_password_hash(user_info.get(username), password)
def networksize_alarm(self, content):
if not content:
return False
node_id = content.get('node_id')
network_id = node_id[:17] # head 8 * 2 bytes
# attention: specially for kroot_id 010000
if network_id.startswith('010000'):
network_id = '010000'
node_id_status = content.get('node_id_status')
if node_id_status == 'remove':
if network_id not in self.network_ids_:
slog.warn('remove node_id:{0} from nonexistent network_id:{1}'.
format(node_id, network_id))
return False
for ni in self.network_ids_[network_id]['node_info']:
if ni.get('node_id') == node_id:
self.network_ids_[network_id]['node_info'].remove(ni)
self.network_ids_[network_id]['size'] -= 1
slog.info(
'remove node_id:{0} from network_id:{1}, now size:{2}'.
format(node_id, network_id,
self.network_ids_[network_id]['size']))
break
return True
if network_id not in self.network_ids_:
network_info = {
'node_info': [{
'node_id': node_id,
'node_ip': content.get('node_ip')
}],
'size':
1,
}
self.network_ids_[network_id] = network_info
slog.info(
'add node_id:{0} to network_id:{1}, new network_id and now size is 1'
.format(node_id, network_id))
return True
else:
for ni in self.network_ids_[network_id]['node_info']:
if ni.get('node_id') == node_id:
#slog.debug('already exist node_id:{0} in network_id:{1}'.format(node_id, network_id))
return True
self.network_ids_[network_id]['node_info'].append({
'node_id':
node_id,
'node_ip':
content.get('node_ip')
})
self.network_ids_[network_id]['size'] += 1
slog.info(
'add node_id:{0} to network_id:{1}, now size is {2}'.format(
node_id, network_id,
self.network_ids_[network_id]['size']))
return True
return True
def alarm_report():
payload = {}
if not request.is_json:
payload = json.loads(request.data)
else:
payload = request.get_json()
ret = {'status': ''}
status_ret = {0: 'OK', -1: '上报字段不合法,部分可能上传失败', -2: '格式转化出错，请检查字段数或者字段格式等'}
if not payload.get('data'):
ret = {'status': -2, 'error': status_ret.get(-2)}
return jsonify(ret)
# TODO(smaug) varify token
alarm_ip = request.headers.get('X-Forwarded-For') or request.headers.get(
'X-Real-IP') or request.remote_addr
slog.info("recv alarm from ip:{0} size:{1}".format(
alarm_ip, len(payload.get('data'))))
mq.handle_alarm(payload.get('data'))
ret = {'status': 0, 'error': status_ret.get(0)}
return jsonify(ret)
def load_db_network_id_num(self):
vs, total = [], 0
vs, total = self.network_id_num_sql_.query_from_db(data={})
if not vs:
slog.warn('load network_id_num from db failed or empty')
return False
for item in vs:
# just for safety check
network_id = item.get('network_id')
network_num = item.get('network_num')
if network_id in self.network_id_num_:
if self.network_id_num_.get(network_id).get(
'network_num') != network_num:
slog.warn(
'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}'
.format(
network_num,
self.network_id_num_.get(network_id).get(
'network_num')))
slog.warn(
'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}'
.format(
network_num,
self.network_id_num_.get(network_id).get(
'network_num')))
slog.warn(
'load network_id_num from db goes wrong, db_network_num:{0} not eq cache_network_num:{1}'
.format(
network_num,
self.network_id_num_.get(network_id).get(
'network_num')))
sys.exit(-1)
self.network_id_num_[item.get('network_id')] = item
slog.info('load network_id_num from db success:{0}'.format(
json.dumps(self.network_id_num_)))
return True
def __init__(self):
# init db obj
self.packet_info_sql = PacketInfoSql()
self.packet_recv_info_sql = PacketRecvInfoSql()
self.network_info_sql = NetworkInfoSql()
self.packet_drop_rate_sql = DropRateInfoSql()
self.node_info_sql_ = NodeInfoSql()
self.system_alarm_info_sql_ = SystemAlarmInfoSql()
self.network_id_num_sql_ = NetworkIdNumSql()
self.system_cron_info_sql_ = SystemCronInfoSql()
self.network_ids_lock_ = threading.Lock()
self.network_ids_ = {}
self.network_id_num_ = {}
self.iplocation_ = {}
self.iplocation_file_ = '/tmp/.topargus_iplocation'
if os.path.exists(self.iplocation_file_):
with open(self.iplocation_file_, 'r') as fin:
self.iplocation_ = json.loads(fin.read())
fin.close()
slog.info('load iplocation from {0}, size:{1}'.format(self.iplocation_file_, len(self.iplocation_.keys())))
return
