def cluster_nameid_from_cache():
objs = tb_fact_nameid_info.objects.all()
dnstype_nameidinfo = {}
for obj in objs:
if obj.nameid_status == "enable" and obj.nameid_name is not None and obj.nameid_policy is not None:
policy = str(obj.nameid_policy)
nameid_str = read_from_cache_cluster("vipdevice", policy,
obj.nameid_name)
if nameid_str != None:
nameid_dict = json.loads(nameid_str)
dnstype = str(obj.dns_type)
if dnstype_nameidinfo.get(dnstype) == None:
dnstype_nameidinfo[dnstype] = {}
dnstype_nameidinfo[dnstype][obj.nameid_name] = nameid_dict
keys = get_keys_from_cache("vipdevice", "qdns-nameidinfo")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if dnstype_nameidinfo.get(key) == None:
logger.info("the dnstype {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "qdns-nameidinfo", key)
for dnstype, nameidinfo in dnstype_nameidinfo.items():
logger.info("the dnstype is {},the nameid info is {}".format(
dnstype, json.dumps(nameidinfo)))
write_to_cache_cluster("vipdevice", "qdns-nameidinfo", str(dnstype),
json.dumps(nameidinfo))
def load_confignameid_from_table():
logger.info("start to init nameid")
objs = tb_fact_nameid_info.objects.all()
current_nameid_dict = {}
for obj in objs:
if obj.nameid_name is not None and obj.nameid_policy is not None and obj.nameid_status == 'enable':
try:
#这个是获取所有view信息的，会以view的id为key,view的元信息为value.是从dimension_view表中取得的信息
url = "{}{}".format("http://127.0.0.1:8000/nameidview/get_info_by_nameid/?nameid=",obj.id)
nameid_view_data = load_data(url)
if nameid_view_data is None or nameid_view_data.get("code") is None or nameid_view_data["code"]==0 or nameid_view_data.get("msg") is None or len(nameid_view_data["msg"]) == 0:
continue
obj_list = nameid_view_data["msg"]
nameid_view_dict = {}
for item in obj_list:
viewobj = ViewClass().genobj(item)
# nameid_view_dict[item["nameid_view_id"]] = viewobj
nameid_view_dict[item["nameid_view_id"]["id"]] = viewobj
logger.info("the nameid is {}.the view is {}".format(obj.nameid_name,json.dumps(nameid_view_dict,default=serialize_instance,ensure_ascii=False)))
#这个是获取所有view和device信息的，是通过 diomension_view_device表中取得的，这里会有详细的nameid_name,device_name，也有view_id,这里会用上面的view_id进行替换。形成最终的信息。
url = "{}{}".format("http://127.0.0.1:8000/nameidviewdevice/get_info_by_nameid/?nameid=",obj.id)
nameid_device_data = load_data(url)
nameidobj = NameidClass()
if nameid_device_data != None and nameid_device_data.get("code") != None and nameid_device_data["code"]==1 and nameid_device_data.get("msg") != None and len(nameid_device_data["msg"]) != 0:
obj_list = nameid_device_data["msg"]
nameid_name = nameidobj.genobj(obj_list,nameid_view_dict)
#这个是获取所有view和cname信息的，是通过dimension_view_cname表中取得的，这里会有详细的cname信息，同样是去填充上面的类中的cname信息。
url = "{}{}".format("http://127.0.0.1:8000/nameidviewcname/get_info_by_nameid/?nameid=",obj.id)
nameid_cname_data = load_data(url)
if nameid_cname_data != None and nameid_cname_data.get("code") != None and nameid_cname_data["code"]==1 and nameid_cname_data.get("msg") != None and len(nameid_cname_data["msg"]) != 0:
obj_list = nameid_cname_data["msg"]
nameid_name = nameidobj.genobj(obj_list,nameid_view_dict)
current_nameid_dict[obj.nameid_name] = nameidobj
#最终nameidobj中存储的就是配置的view和每个view中对应的设备或者cname信息。后面的都是基于此来做解析的。会以nameid_name为key,它的元信息为value存入到cache中
#write_to_cache_cluster("vipdevice","nameid-manual",obj.nameid_name,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance))
#logger.info("the nameid is {},the config is {}".format(obj.nameid_name,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance)))
except Exception as err:
logger.error(err)
continue
keys = get_keys_from_cache("vipdevice","nameid-manual")
for key in keys:
key = str(key,encoding = "raw_unicode_escape")
if current_nameid_dict.get(key) == None:
logger.info("the nameid {} is disable".format(key))
delete_to_cache_cluster("vipdevice","nameid-manual",key)
for nameid,nameidobj in current_nameid_dict.items():
#最终nameidobj中存储的就是配置的view和每个view中对应的设备或者cname信息。后面的都是基于此来做解析的。会以nameid_name为key,它的元信息为value存入到cache中
write_to_cache_cluster("vipdevice","nameid-manual",nameid,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance))
logger.info("the nameid is {},the config is {}".format(nameid,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance,ensure_ascii=False)))
def gen_detect_viplist_cache():
try:
sql = "select detect_frency,detect_frency_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
res = my_custom_sql(sql)
frequency = 15
frequency_unit = 'second'
for item in res:
frequency = item[0]
frequency_unit = item[1]
if frequency == None or frequency_unit == None:
frequency = 15
frequency_unit = 'second'
sql = "select a.admin_ip,b.vip_address,a.isp,a.availability_status from Polaris_tb_fact_adminip_info as a left join Polaris_tb_fact_device_info as b on a.isp =b.node_isp"
res = my_custom_sql(sql)
raw_vip = {}
for item in res:
adminip = item[0]
vip = item[1]
isp = item[2]
status = item[3]
if raw_vip.get(adminip) == None:
raw_vip[adminip] = {}
raw_vip[adminip]["vip_address"] = []
raw_vip[adminip]["adminip_isp"] = isp
raw_vip[adminip]["detect_switch"] = status
raw_vip[adminip]["frequency"] = frequency
# raw_vip[adminip]["frequency_unit"] = frequency_unit
raw_vip[adminip]["vip_address"].append(vip)
keys = get_keys_from_cache("vipdevice", "detect-vipaddress")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if raw_vip.get(key) == None:
logger.info("the adminip {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "detect-vipaddress", key)
for item in raw_vip:
write_to_cache_cluster("vipdevice", "detect-vipaddress", item,
json.dumps(raw_vip[item]))
logger.info("adminip is {},detect vip is {} ".format(
item,
read_from_cache_cluster("vipdevice", "detect-vipaddress",
item)))
except Exception as err:
logger.error(err)
def generate_device_status_cache():
try:
sql = 'select vip_address,vip_status,vip_enable_switch from Polaris_tb_fact_device_info'
res = my_custom_sql(sql)
vip_avil_dict = {}
for item in res:
vip_address = item[0]
vip_status = item[1]
vip_enable_switch = item[2]
if vip_enable_switch == "enable":
detect_vip_availability = read_from_cache_cluster(
"vipdevice", "detect-vipaddress-availability", vip_address)
if detect_vip_availability == "enable" or detect_vip_availability == "disable":
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,detect_vip_availability)
vip_avil_dict[vip_address] = detect_vip_availability
else:
vip_avil_dict[vip_address] = vip_status
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
else:
vip_avil_dict[vip_address] = vip_status
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
logger.info(
"the vip {},the vip_enable_switch is {},the vip_status is {},the detect status is {}"
.format(
vip_address, vip_enable_switch, vip_status,
read_from_cache_cluster("vipdevice",
"vipaddress-availability",
vip_address)))
keys = get_keys_from_cache("vipdevice", "vipaddress-availability")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if vip_avil_dict.get(key) == None:
logger.info("the vip {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "vipaddress-availability",
key)
for vip, status in vip_avil_dict.items():
write_to_cache_cluster("vipdevice", "vipaddress-availability", vip,
status)
except Exception as err:
logger.error(err)
def load_zone_from_table():
zoneqdns_dict = {}
sql = 'select zone_name,record_name,record_content,internet_type,record_type,record_ttl,dns_name from Polaris_tb_fact_backend_dnszone_info left join Polaris_tb_fact_zonetype_info on Polaris_tb_fact_backend_dnszone_info.zone_name_id = Polaris_tb_fact_zonetype_info.id left join Polaris_tb_fact_dnstype_info on Polaris_tb_fact_backend_dnszone_info.dns_type_id = Polaris_tb_fact_dnstype_info.id where zone_status = "enable"'
zoneres = my_custom_sql(sql)
if zoneres != None and len(zoneres) != 0:
zonename_tag = False
for zoneitem in zoneres:
zone_name = zoneitem[0]
record_name = zoneitem[1]
record_content = zoneitem[2]
internet_type = zoneitem[3]
record_type = zoneitem[4]
record_ttl = zoneitem[5]
dnsname = zoneitem[6]
if zoneqdns_dict.get(dnsname) == None:
zoneqdns_dict[dnsname] = {}
if zoneqdns_dict[dnsname].get(zone_name) == None:
zoneqdns_dict[dnsname][zone_name] = []
zoneqdns_dict[dnsname][zone_name].append({
"record_name": record_name,
"record_content": record_content,
"internet_type": internet_type,
"record_type": record_type,
"record_ttl": record_ttl
})
# zoneqdns_dict[dnsname][zone_name].append("{} {} {} {} {}\n".format(record_name,record_ttl,internet_type,record_type,record_content))
if len(zoneqdns_dict) == 0:
logger.info("the zone dict is null")
keys = get_keys_from_cache("vipdevice", "zone-config")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if zoneqdns_dict.get(key) == None:
logger.info("the nameid {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "zone-config", key)
for dnstype, zoneinfo in zoneqdns_dict.items():
logger.info("the dnstype is {},the zone info is {}".format(
dnstype, json.dumps(zoneinfo)))
write_to_cache_cluster("vipdevice", "zone-config", str(dnstype),
json.dumps(zoneinfo))
def sload_zone_from_table():
zoneqdns_dict = {}
objs = tb_fact_nameid_info.objects.all()
for obj in objs:
nameid = obj.nameid_name
status = obj.nameid_status
if status != "enable":
continue
sql = 'select a.dns_type_id,b.dns_name from Polaris_tb_fact_nameid_info a left join Polaris_tb_fact_dnstype_info b on a.dns_type_id = b.id where nameid_name = "{}"'.format(
nameid)
res = my_custom_sql(sql)
if res != None and len(res) != 0:
for item in res:
id = item[0]
dnsname = item[1]
sql = 'select zone_name,record_name,record_content,internet_type,record_type,record_ttl from Polaris_tb_fact_dnszone_info left join Polaris_tb_fact_zonetype_info on Polaris_tb_fact_dnszone_info.zone_name_id = Polaris_tb_fact_zonetype_info.id where zone_status = "enable" and dns_type_id = "{}"'.format(
id)
zoneres = my_custom_sql(sql)
if zoneres != None and len(zoneres) != 0:
zonename_tag = False
for zoneitem in zoneres:
zone_name = zoneitem[0]
if nameid.find(zone_name) != -1:
zonename_tag = True
record_name = zoneitem[1]
record_content = zoneitem[2]
internet_type = zoneitem[3]
record_type = zoneitem[4]
record_ttl = zoneitem[5]
if zoneqdns_dict.get(dnsname) == None:
zoneqdns_dict[dnsname] = {}
if zoneqdns_dict[dnsname].get(zone_name) == None:
zoneqdns_dict[dnsname][zone_name] = []
zoneqdns_dict[dnsname][zone_name].append({
"record_name":
record_name,
"record_content":
record_content,
"internet_type":
internet_type,
"record_type":
record_type,
"record_ttl":
record_ttl
})
if len(zoneqdns_dict) == 0 or zonename_tag == False:
logger.info(
"the nameid zone is not legal,the nameid is {}".
format(nameid))
if len(zoneqdns_dict) == 0:
logger.info("the zone dict is null")
keys = get_keys_from_cache("vipdevice", "zone-config")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if zoneqdns_dict.get(key) == None:
logger.info("the nameid {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "zone-config", key)
for dnstype, zoneinfo in zoneqdns_dict.items():
logger.info("the dnstype is {},the zone info is {}".format(
dnstype, json.dumps(zoneinfo)))
write_to_cache_cluster("vipdevice", "zone-config", str(dnstype),
json.dumps(zoneinfo))
def load_device_availability_cache():
try:
sql = "select node_isp,total_value,absolute_value,relative_rate from Polaris_tb_fact_detectdeviceavailability_standard_info"
res = my_custom_sql(sql)
isp_standard_data = {}
for item in res:
isp = item[0]
total_value = item[1]
absolute_value = item[2]
relative_rate = item[3]
if isp_standard_data.get(isp) == None:
isp_standard_data[isp] = {}
isp_standard_data[isp]["total_value"] = total_value
isp_standard_data[isp]["absolute_value"] = absolute_value
isp_standard_data[isp]["relative_rate"] = relative_rate
sql = "select effective_time,effective_time_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
res = my_custom_sql(sql)
effective_time = 15
effective_time_unit = 'second'
for item in res:
effective_time = item[0]
effective_time_unit = item[1]
if effective_time == None or effective_time_unit == None:
effective_time = 15
effective_time_unit = 'second'
#这里group by vip和availability，以vip为key,拿到最终上报的关于该vip的总的探针个数，上报的状态，以及运营商
sql = 'select vip_address,availability,admin_isp,count(availability) from Polaris_tb_fact_detectdeviceavailability_select_info where create_time >= DATE_FORMAT(DATE_ADD(now(), INTERVAL - {} {}),"%y-%m-%d %H:%i:%s") group by vip_address,availability'.format(
effective_time, effective_time_unit)
res = my_custom_sql(sql)
vip_avil_dict = {}
for item in res:
vip_address = item[0]
status = item[1]
admin_isp = item[2]
num = item[3]
#设备的运营商信息以最后一次为准
if vip_avil_dict.get(vip_address) == None:
vip_avil_dict[vip_address] = [status, num, admin_isp]
elif vip_avil_dict[vip_address][1] < num:
vip_avil_dict[vip_address][0] = status
vip_avil_dict[vip_address][
1] = vip_avil_dict[vip_address][1] + num
vip_avil_dict[vip_address][2] = admin_isp
else:
vip_avil_dict[vip_address][
1] = vip_avil_dict[vip_address][1] + num
vip_avil_dict[vip_address][2] = admin_isp
detect_vip_status = {}
#if vip_avil_dict.get(vip_address) == None or vip_avil_dict[vip_address][1] < num:
# vip_avil_dict[vip_address] = [status,num,admin_isp]
#开始确定每个vip的状态,对于每个vip都要满足俩个标准，一个是上报的探针个数要达到那个绝对值，还有一个是要达到那个相对值,会有三种状态，对于数据不符合检验标准的，都认为是无效的，对于无效的探测数据，还是要采用手工配置的方式
for item in vip_avil_dict:
status = vip_avil_dict[item][0]
num = vip_avil_dict[item][1]
admin_isp = vip_avil_dict[item][2]
if isp_standard_data.get(admin_isp) == None:
detect_vip_status[item] = "invalid"
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
else:
total_value = isp_standard_data[admin_isp]["total_value"]
absolute_value = isp_standard_data[admin_isp]["absolute_value"]
relative_rate = isp_standard_data[admin_isp]["relative_rate"]
logger.info(
"the vip is {},the total detect alive vip num is {},the standared num is {},the relative percent alive is {},the standared is {}"
.format(item, num, absolute_value, num / total_value,
relative_rate))
if num < absolute_value or (num / total_value < relative_rate):
detect_vip_status[item] = "invalid"
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
else:
detect_vip_status[item] = vip_avil_dict[item][0]
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,vip_avil_dict[item][0])
logger.info("the vip {},the detect status is {}".format(
item,
read_from_cache_cluster("vipdevice",
"detect-vipaddress-availability",
item)))
keys = get_keys_from_cache("vipdevice",
"detect-vipaddress-availability")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if detect_vip_status.get(key) == None:
logger.info("the detect vip {} is disable".format(key))
delete_to_cache_cluster("vipdevice",
"detect-vipaddress-availability", key)
for vip, status in detect_vip_status.items():
write_to_cache_cluster("vipdevice",
"detect-vipaddress-availability", vip,
status)
except Exception as err:
logger.error(err)
def vipdevice_availability_policy(multiple_config):
qdns_res = {}
policy_key = ""
if multiple_config != "nameid-default" and multiple_config != "nameid-manual":
return
objs = tb_fact_nameid_info.objects.all()
for obj in objs:
if obj.nameid_name is not None:
try:
nameid = obj.nameid_name
status = obj.nameid_status
policy = str(obj.nameid_policy)
policy_key = policy
dnszone = str(obj.zone_type.zone_name)
if not nameid.endswith(dnszone):
logger.info(
"the nameid {} is not legal...".format(nameid))
continue
nameid_obj = read_from_cache_cluster("vipdevice",
multiple_config, nameid)
if nameid_obj != None:
nameid_obj = json.loads(nameid_obj)
nameid_obj_replace = {}
for viewname, viewinfo in nameid_obj.items():
if viewinfo != None:
nameid_obj_replace[viewname] = {}
for k, v in viewinfo.items():
if k == "address":
nameid_obj_replace[viewname][k] = {}
for vip, ratio in v.items():
vipaddress_availability = read_from_cache_cluster(
"vipdevice",
"vipaddress-availability", vip)
if vipaddress_availability != None:
if vipaddress_availability == "enable":
nameid_obj_replace[viewname][
k][vip] = ratio
else:
nameid_obj_replace[viewname][k] = v
if (
nameid_obj_replace[viewname].get("address")
== None and
nameid_obj_replace[viewname].get("cname")
== None
) or (nameid_obj_replace[viewname].get("address")
!= None and len(
nameid_obj_replace[viewname]["address"])
== 0
and nameid_obj_replace[viewname].get("cname")
!= None and len(
nameid_obj_replace[viewname]["cname"])
== 0):
del nameid_obj_replace[viewname]
logger.info(
"the nameid is {},the view {} is null".
format(nameid, viewname))
if len(nameid_obj_replace) != 0:
if qdns_res.get(nameid) == None:
qdns_res[nameid] = {}
qdns_res[nameid] = nameid_obj_replace
except Exception as err:
logger.error(err)
continue
if policy_key == None:
policy_key = "vipdevice_availability_policy"
keys = get_keys_from_cache("vipdevice", policy_key)
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if qdns_res.get(key) == None:
logger.info("the nameid {} is disable".format(key))
delete_to_cache_cluster("vipdevice", policy_key, key)
for nameid, nameidinfo in qdns_res.items():
logger.info("the nameid is {},the nameid info is {}".format(
nameid, json.dumps(nameidinfo)))
write_to_cache_cluster("vipdevice", "vipdevice_availability_policy",
nameid, json.dumps(nameidinfo))