def cluster_nameid_from_cache():
objs = tb_fact_nameid_info.objects.all()
dnstype_nameidinfo = {}
for obj in objs:
if obj.nameid_status == "enable" and obj.nameid_name is not None and obj.nameid_policy is not None:
policy = str(obj.nameid_policy)
nameid_str = read_from_cache_cluster("vipdevice", policy,
obj.nameid_name)
if nameid_str != None:
nameid_dict = json.loads(nameid_str)
dnstype = str(obj.dns_type)
if dnstype_nameidinfo.get(dnstype) == None:
dnstype_nameidinfo[dnstype] = {}
dnstype_nameidinfo[dnstype][obj.nameid_name] = nameid_dict
keys = get_keys_from_cache("vipdevice", "qdns-nameidinfo")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if dnstype_nameidinfo.get(key) == None:
logger.info("the dnstype {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "qdns-nameidinfo", key)
for dnstype, nameidinfo in dnstype_nameidinfo.items():
logger.info("the dnstype is {},the nameid info is {}".format(
dnstype, json.dumps(nameidinfo)))
write_to_cache_cluster("vipdevice", "qdns-nameidinfo", str(dnstype),
json.dumps(nameidinfo))
def generate_device_status_cache():
try:
sql = 'select vip_address,vip_status,vip_enable_switch from Polaris_tb_fact_device_info'
res = my_custom_sql(sql)
vip_avil_dict = {}
for item in res:
vip_address = item[0]
vip_status = item[1]
vip_enable_switch = item[2]
if vip_enable_switch == "enable":
detect_vip_availability = read_from_cache_cluster(
"vipdevice", "detect-vipaddress-availability", vip_address)
if detect_vip_availability == "enable" or detect_vip_availability == "disable":
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,detect_vip_availability)
vip_avil_dict[vip_address] = detect_vip_availability
else:
vip_avil_dict[vip_address] = vip_status
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
else:
vip_avil_dict[vip_address] = vip_status
#write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
logger.info(
"the vip {},the vip_enable_switch is {},the vip_status is {},the detect status is {}"
.format(
vip_address, vip_enable_switch, vip_status,
read_from_cache_cluster("vipdevice",
"vipaddress-availability",
vip_address)))
keys = get_keys_from_cache("vipdevice", "vipaddress-availability")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if vip_avil_dict.get(key) == None:
logger.info("the vip {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "vipaddress-availability",
key)
for vip, status in vip_avil_dict.items():
write_to_cache_cluster("vipdevice", "vipaddress-availability", vip,
status)
except Exception as err:
logger.error(err)
def gen_detect_viplist_cache():
try:
sql = "select detect_frency,detect_frency_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
res = my_custom_sql(sql)
frequency = 15
frequency_unit = 'second'
for item in res:
frequency = item[0]
frequency_unit = item[1]
if frequency == None or frequency_unit == None:
frequency = 15
frequency_unit = 'second'
sql = "select a.admin_ip,b.vip_address,a.isp,a.availability_status from Polaris_tb_fact_adminip_info as a left join Polaris_tb_fact_device_info as b on a.isp =b.node_isp"
res = my_custom_sql(sql)
raw_vip = {}
for item in res:
adminip = item[0]
vip = item[1]
isp = item[2]
status = item[3]
if raw_vip.get(adminip) == None:
raw_vip[adminip] = {}
raw_vip[adminip]["vip_address"] = []
raw_vip[adminip]["adminip_isp"] = isp
raw_vip[adminip]["detect_switch"] = status
raw_vip[adminip]["frequency"] = frequency
# raw_vip[adminip]["frequency_unit"] = frequency_unit
raw_vip[adminip]["vip_address"].append(vip)
keys = get_keys_from_cache("vipdevice", "detect-vipaddress")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if raw_vip.get(key) == None:
logger.info("the adminip {} is disable".format(key))
delete_to_cache_cluster("vipdevice", "detect-vipaddress", key)
for item in raw_vip:
write_to_cache_cluster("vipdevice", "detect-vipaddress", item,
json.dumps(raw_vip[item]))
logger.info("adminip is {},detect vip is {} ".format(
item,
read_from_cache_cluster("vipdevice", "detect-vipaddress",
item)))
except Exception as err:
logger.error(err)
def get_vipaddress_from_cache(address):
logger.info("the adminip {} get task ".format(address))
res = read_from_cache_cluster("vipdevice", "detect-vipaddress", address)
if res == None:
return ""
return res
def node_best_qos_policy(nameid,policy):
logger.info(nameid)
res = read_from_cache_cluster("vipdevice","vipdevice_availability",nameid)
logger.info(res)
def get_zone_from_cache(dnstype):
logger.info("the {} qdns get zone ".format(dnstype))
res = read_from_cache_cluster("vipdevice", "zone-config", dnstype)
if res == None:
return ""
return res
def load_device_availability_cache():
try:
sql = "select node_isp,total_value,absolute_value,relative_rate from Polaris_tb_fact_detectdeviceavailability_standard_info"
res = my_custom_sql(sql)
isp_standard_data = {}
for item in res:
isp = item[0]
total_value = item[1]
absolute_value = item[2]
relative_rate = item[3]
if isp_standard_data.get(isp) == None:
isp_standard_data[isp] = {}
isp_standard_data[isp]["total_value"] = total_value
isp_standard_data[isp]["absolute_value"] = absolute_value
isp_standard_data[isp]["relative_rate"] = relative_rate
sql = "select effective_time,effective_time_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
res = my_custom_sql(sql)
effective_time = 15
effective_time_unit = 'second'
for item in res:
effective_time = item[0]
effective_time_unit = item[1]
if effective_time == None or effective_time_unit == None:
effective_time = 15
effective_time_unit = 'second'
#这里group by vip和availability，以vip为key,拿到最终上报的关于该vip的总的探针个数，上报的状态，以及运营商
sql = 'select vip_address,availability,admin_isp,count(availability) from Polaris_tb_fact_detectdeviceavailability_select_info where create_time >= DATE_FORMAT(DATE_ADD(now(), INTERVAL - {} {}),"%y-%m-%d %H:%i:%s") group by vip_address,availability'.format(
effective_time, effective_time_unit)
res = my_custom_sql(sql)
vip_avil_dict = {}
for item in res:
vip_address = item[0]
status = item[1]
admin_isp = item[2]
num = item[3]
#设备的运营商信息以最后一次为准
if vip_avil_dict.get(vip_address) == None:
vip_avil_dict[vip_address] = [status, num, admin_isp]
elif vip_avil_dict[vip_address][1] < num:
vip_avil_dict[vip_address][0] = status
vip_avil_dict[vip_address][
1] = vip_avil_dict[vip_address][1] + num
vip_avil_dict[vip_address][2] = admin_isp
else:
vip_avil_dict[vip_address][
1] = vip_avil_dict[vip_address][1] + num
vip_avil_dict[vip_address][2] = admin_isp
detect_vip_status = {}
#if vip_avil_dict.get(vip_address) == None or vip_avil_dict[vip_address][1] < num:
# vip_avil_dict[vip_address] = [status,num,admin_isp]
#开始确定每个vip的状态,对于每个vip都要满足俩个标准，一个是上报的探针个数要达到那个绝对值，还有一个是要达到那个相对值,会有三种状态，对于数据不符合检验标准的，都认为是无效的，对于无效的探测数据，还是要采用手工配置的方式
for item in vip_avil_dict:
status = vip_avil_dict[item][0]
num = vip_avil_dict[item][1]
admin_isp = vip_avil_dict[item][2]
if isp_standard_data.get(admin_isp) == None:
detect_vip_status[item] = "invalid"
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
else:
total_value = isp_standard_data[admin_isp]["total_value"]
absolute_value = isp_standard_data[admin_isp]["absolute_value"]
relative_rate = isp_standard_data[admin_isp]["relative_rate"]
logger.info(
"the vip is {},the total detect alive vip num is {},the standared num is {},the relative percent alive is {},the standared is {}"
.format(item, num, absolute_value, num / total_value,
relative_rate))
if num < absolute_value or (num / total_value < relative_rate):
detect_vip_status[item] = "invalid"
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
else:
detect_vip_status[item] = vip_avil_dict[item][0]
#write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,vip_avil_dict[item][0])
logger.info("the vip {},the detect status is {}".format(
item,
read_from_cache_cluster("vipdevice",
"detect-vipaddress-availability",
item)))
keys = get_keys_from_cache("vipdevice",
"detect-vipaddress-availability")
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if detect_vip_status.get(key) == None:
logger.info("the detect vip {} is disable".format(key))
delete_to_cache_cluster("vipdevice",
"detect-vipaddress-availability", key)
for vip, status in detect_vip_status.items():
write_to_cache_cluster("vipdevice",
"detect-vipaddress-availability", vip,
status)
except Exception as err:
logger.error(err)
def get_nameid_from_cache(dnstype):
logger.info("the {} qdns get nameid ".format(dnstype))
res = read_from_cache_cluster("vipdevice", "qdns-nameidinfo", dnstype)
if res == None:
return ""
return res
def vipdevice_availability_policy(multiple_config):
qdns_res = {}
policy_key = ""
if multiple_config != "nameid-default" and multiple_config != "nameid-manual":
return
objs = tb_fact_nameid_info.objects.all()
for obj in objs:
if obj.nameid_name is not None:
try:
nameid = obj.nameid_name
status = obj.nameid_status
policy = str(obj.nameid_policy)
policy_key = policy
dnszone = str(obj.zone_type.zone_name)
if not nameid.endswith(dnszone):
logger.info(
"the nameid {} is not legal...".format(nameid))
continue
nameid_obj = read_from_cache_cluster("vipdevice",
multiple_config, nameid)
if nameid_obj != None:
nameid_obj = json.loads(nameid_obj)
nameid_obj_replace = {}
for viewname, viewinfo in nameid_obj.items():
if viewinfo != None:
nameid_obj_replace[viewname] = {}
for k, v in viewinfo.items():
if k == "address":
nameid_obj_replace[viewname][k] = {}
for vip, ratio in v.items():
vipaddress_availability = read_from_cache_cluster(
"vipdevice",
"vipaddress-availability", vip)
if vipaddress_availability != None:
if vipaddress_availability == "enable":
nameid_obj_replace[viewname][
k][vip] = ratio
else:
nameid_obj_replace[viewname][k] = v
if (
nameid_obj_replace[viewname].get("address")
== None and
nameid_obj_replace[viewname].get("cname")
== None
) or (nameid_obj_replace[viewname].get("address")
!= None and len(
nameid_obj_replace[viewname]["address"])
== 0
and nameid_obj_replace[viewname].get("cname")
!= None and len(
nameid_obj_replace[viewname]["cname"])
== 0):
del nameid_obj_replace[viewname]
logger.info(
"the nameid is {},the view {} is null".
format(nameid, viewname))
if len(nameid_obj_replace) != 0:
if qdns_res.get(nameid) == None:
qdns_res[nameid] = {}
qdns_res[nameid] = nameid_obj_replace
except Exception as err:
logger.error(err)
continue
if policy_key == None:
policy_key = "vipdevice_availability_policy"
keys = get_keys_from_cache("vipdevice", policy_key)
for key in keys:
key = str(key, encoding="raw_unicode_escape")
if qdns_res.get(key) == None:
logger.info("the nameid {} is disable".format(key))
delete_to_cache_cluster("vipdevice", policy_key, key)
for nameid, nameidinfo in qdns_res.items():
logger.info("the nameid is {},the nameid info is {}".format(
nameid, json.dumps(nameidinfo)))
write_to_cache_cluster("vipdevice", "vipdevice_availability_policy",
nameid, json.dumps(nameidinfo))