示例#1
0
def cluster_nameid_from_cache():
    objs = tb_fact_nameid_info.objects.all()
    dnstype_nameidinfo = {}
    for obj in objs:
        if obj.nameid_status == "enable" and obj.nameid_name is not None and obj.nameid_policy is not None:
            policy = str(obj.nameid_policy)
            nameid_str = read_from_cache_cluster("vipdevice", policy,
                                                 obj.nameid_name)
            if nameid_str != None:
                nameid_dict = json.loads(nameid_str)
                dnstype = str(obj.dns_type)
                if dnstype_nameidinfo.get(dnstype) == None:
                    dnstype_nameidinfo[dnstype] = {}
                dnstype_nameidinfo[dnstype][obj.nameid_name] = nameid_dict
    keys = get_keys_from_cache("vipdevice", "qdns-nameidinfo")
    for key in keys:
        key = str(key, encoding="raw_unicode_escape")
        if dnstype_nameidinfo.get(key) == None:
            logger.info("the dnstype {} is disable".format(key))
            delete_to_cache_cluster("vipdevice", "qdns-nameidinfo", key)
    for dnstype, nameidinfo in dnstype_nameidinfo.items():
        logger.info("the dnstype is {},the nameid info is {}".format(
            dnstype, json.dumps(nameidinfo)))
        write_to_cache_cluster("vipdevice", "qdns-nameidinfo", str(dnstype),
                               json.dumps(nameidinfo))
示例#2
0
def load_confignameid_from_table():
    logger.info("start to init nameid")
    objs = tb_fact_nameid_info.objects.all()
    current_nameid_dict = {}
    for obj in objs:
        if obj.nameid_name is not None and obj.nameid_policy is not None and obj.nameid_status == 'enable':
            try:
                #这个是获取所有view信息的,会以view的id为key,view的元信息为value.是从dimension_view表中取得的信息
                url = "{}{}".format("http://127.0.0.1:8000/nameidview/get_info_by_nameid/?nameid=",obj.id)
                nameid_view_data = load_data(url)
                if nameid_view_data is None or nameid_view_data.get("code") is None or nameid_view_data["code"]==0 or nameid_view_data.get("msg") is None or len(nameid_view_data["msg"]) == 0:
                    continue
                obj_list = nameid_view_data["msg"]
                nameid_view_dict = {}
                for item in obj_list:
                    viewobj = ViewClass().genobj(item)
                   # nameid_view_dict[item["nameid_view_id"]] = viewobj
                    nameid_view_dict[item["nameid_view_id"]["id"]] = viewobj
                logger.info("the nameid is {}.the view is {}".format(obj.nameid_name,json.dumps(nameid_view_dict,default=serialize_instance,ensure_ascii=False)))
                #这个是获取所有view和device信息的,是通过 diomension_view_device表中取得的,这里会有详细的nameid_name,device_name,也有view_id,这里会用上面的view_id进行替换。形成最终的信息。
                url = "{}{}".format("http://127.0.0.1:8000/nameidviewdevice/get_info_by_nameid/?nameid=",obj.id)
                nameid_device_data = load_data(url)
                nameidobj = NameidClass()  
                if nameid_device_data != None and nameid_device_data.get("code") != None and nameid_device_data["code"]==1 and nameid_device_data.get("msg") != None and len(nameid_device_data["msg"]) != 0:  
                    obj_list = nameid_device_data["msg"]
                    nameid_name = nameidobj.genobj(obj_list,nameid_view_dict)
                #这个是获取所有view和cname信息的,是通过dimension_view_cname表中取得的,这里会有详细的cname信息,同样是去填充上面的类中的cname信息。
                url = "{}{}".format("http://127.0.0.1:8000/nameidviewcname/get_info_by_nameid/?nameid=",obj.id)
                nameid_cname_data = load_data(url)
                if nameid_cname_data != None and nameid_cname_data.get("code") != None and nameid_cname_data["code"]==1 and nameid_cname_data.get("msg") != None and len(nameid_cname_data["msg"]) != 0:  
                    obj_list = nameid_cname_data["msg"]
                    nameid_name = nameidobj.genobj(obj_list,nameid_view_dict)
                current_nameid_dict[obj.nameid_name] = nameidobj
                #最终nameidobj中存储的就是配置的view和每个view中对应的设备或者cname信息。后面的都是基于此来做解析的。会以nameid_name为key,它的元信息为value存入到cache中
                #write_to_cache_cluster("vipdevice","nameid-manual",obj.nameid_name,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance))
                #logger.info("the nameid is {},the config is {}".format(obj.nameid_name,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance))) 
            except Exception as err:
                logger.error(err)
                continue
    keys = get_keys_from_cache("vipdevice","nameid-manual")   
    for key in keys:
        key = str(key,encoding = "raw_unicode_escape")
        if current_nameid_dict.get(key) == None:
            logger.info("the nameid {} is disable".format(key))
            delete_to_cache_cluster("vipdevice","nameid-manual",key)
    for nameid,nameidobj in current_nameid_dict.items():
        #最终nameidobj中存储的就是配置的view和每个view中对应的设备或者cname信息。后面的都是基于此来做解析的。会以nameid_name为key,它的元信息为value存入到cache中
        write_to_cache_cluster("vipdevice","nameid-manual",nameid,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance))
        logger.info("the nameid is {},the config is {}".format(nameid,json.dumps(nameidobj.nameid_data_dict,default=serialize_instance,ensure_ascii=False))) 
def gen_detect_viplist_cache():
    try:
        sql = "select detect_frency,detect_frency_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
        res = my_custom_sql(sql)
        frequency = 15
        frequency_unit = 'second'
        for item in res:
            frequency = item[0]
            frequency_unit = item[1]
            if frequency == None or frequency_unit == None:
                frequency = 15
                frequency_unit = 'second'
        sql = "select a.admin_ip,b.vip_address,a.isp,a.availability_status from Polaris_tb_fact_adminip_info as a left join Polaris_tb_fact_device_info as b on a.isp =b.node_isp"
        res = my_custom_sql(sql)
        raw_vip = {}
        for item in res:
            adminip = item[0]
            vip = item[1]
            isp = item[2]
            status = item[3]
            if raw_vip.get(adminip) == None:
                raw_vip[adminip] = {}
                raw_vip[adminip]["vip_address"] = []
                raw_vip[adminip]["adminip_isp"] = isp
                raw_vip[adminip]["detect_switch"] = status
                raw_vip[adminip]["frequency"] = frequency
            #    raw_vip[adminip]["frequency_unit"] = frequency_unit

            raw_vip[adminip]["vip_address"].append(vip)
        keys = get_keys_from_cache("vipdevice", "detect-vipaddress")
        for key in keys:
            key = str(key, encoding="raw_unicode_escape")
            if raw_vip.get(key) == None:
                logger.info("the adminip {} is disable".format(key))
                delete_to_cache_cluster("vipdevice", "detect-vipaddress", key)
        for item in raw_vip:
            write_to_cache_cluster("vipdevice", "detect-vipaddress", item,
                                   json.dumps(raw_vip[item]))
            logger.info("adminip is {},detect vip is {} ".format(
                item,
                read_from_cache_cluster("vipdevice", "detect-vipaddress",
                                        item)))
    except Exception as err:
        logger.error(err)
def generate_device_status_cache():
    try:
        sql = 'select vip_address,vip_status,vip_enable_switch from Polaris_tb_fact_device_info'
        res = my_custom_sql(sql)
        vip_avil_dict = {}
        for item in res:
            vip_address = item[0]
            vip_status = item[1]
            vip_enable_switch = item[2]

            if vip_enable_switch == "enable":
                detect_vip_availability = read_from_cache_cluster(
                    "vipdevice", "detect-vipaddress-availability", vip_address)
                if detect_vip_availability == "enable" or detect_vip_availability == "disable":
                    #write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,detect_vip_availability)
                    vip_avil_dict[vip_address] = detect_vip_availability
                else:
                    vip_avil_dict[vip_address] = vip_status
                    #write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
            else:
                vip_avil_dict[vip_address] = vip_status
                #write_to_cache_cluster("vipdevice","vipaddress-availability",vip_address,vip_status)
            logger.info(
                "the vip {},the vip_enable_switch is {},the vip_status is {},the detect status is {}"
                .format(
                    vip_address, vip_enable_switch, vip_status,
                    read_from_cache_cluster("vipdevice",
                                            "vipaddress-availability",
                                            vip_address)))
        keys = get_keys_from_cache("vipdevice", "vipaddress-availability")
        for key in keys:
            key = str(key, encoding="raw_unicode_escape")
            if vip_avil_dict.get(key) == None:
                logger.info("the vip {} is disable".format(key))
                delete_to_cache_cluster("vipdevice", "vipaddress-availability",
                                        key)
        for vip, status in vip_avil_dict.items():
            write_to_cache_cluster("vipdevice", "vipaddress-availability", vip,
                                   status)
    except Exception as err:
        logger.error(err)
示例#5
0
def load_zone_from_table():
    zoneqdns_dict = {}
    sql = 'select zone_name,record_name,record_content,internet_type,record_type,record_ttl,dns_name from Polaris_tb_fact_backend_dnszone_info left join Polaris_tb_fact_zonetype_info on Polaris_tb_fact_backend_dnszone_info.zone_name_id = Polaris_tb_fact_zonetype_info.id left join Polaris_tb_fact_dnstype_info on Polaris_tb_fact_backend_dnszone_info.dns_type_id = Polaris_tb_fact_dnstype_info.id where zone_status = "enable"'
    zoneres = my_custom_sql(sql)
    if zoneres != None and len(zoneres) != 0:
        zonename_tag = False
        for zoneitem in zoneres:
            zone_name = zoneitem[0]
            record_name = zoneitem[1]
            record_content = zoneitem[2]
            internet_type = zoneitem[3]
            record_type = zoneitem[4]
            record_ttl = zoneitem[5]
            dnsname = zoneitem[6]
            if zoneqdns_dict.get(dnsname) == None:
                zoneqdns_dict[dnsname] = {}
            if zoneqdns_dict[dnsname].get(zone_name) == None:
                zoneqdns_dict[dnsname][zone_name] = []
            zoneqdns_dict[dnsname][zone_name].append({
                "record_name": record_name,
                "record_content": record_content,
                "internet_type": internet_type,
                "record_type": record_type,
                "record_ttl": record_ttl
            })
        #  zoneqdns_dict[dnsname][zone_name].append("{} {} {} {} {}\n".format(record_name,record_ttl,internet_type,record_type,record_content))
    if len(zoneqdns_dict) == 0:
        logger.info("the zone dict is null")
    keys = get_keys_from_cache("vipdevice", "zone-config")
    for key in keys:
        key = str(key, encoding="raw_unicode_escape")
        if zoneqdns_dict.get(key) == None:
            logger.info("the nameid {} is disable".format(key))
            delete_to_cache_cluster("vipdevice", "zone-config", key)
    for dnstype, zoneinfo in zoneqdns_dict.items():
        logger.info("the dnstype is {},the zone info is {}".format(
            dnstype, json.dumps(zoneinfo)))
        write_to_cache_cluster("vipdevice", "zone-config", str(dnstype),
                               json.dumps(zoneinfo))
示例#6
0
def sload_zone_from_table():
    zoneqdns_dict = {}
    objs = tb_fact_nameid_info.objects.all()
    for obj in objs:
        nameid = obj.nameid_name
        status = obj.nameid_status
        if status != "enable":
            continue
        sql = 'select a.dns_type_id,b.dns_name from Polaris_tb_fact_nameid_info a left join Polaris_tb_fact_dnstype_info b on a.dns_type_id = b.id where nameid_name = "{}"'.format(
            nameid)
        res = my_custom_sql(sql)
        if res != None and len(res) != 0:
            for item in res:
                id = item[0]
                dnsname = item[1]
                sql = 'select zone_name,record_name,record_content,internet_type,record_type,record_ttl from Polaris_tb_fact_dnszone_info left join Polaris_tb_fact_zonetype_info on Polaris_tb_fact_dnszone_info.zone_name_id = Polaris_tb_fact_zonetype_info.id where zone_status = "enable" and dns_type_id = "{}"'.format(
                    id)
                zoneres = my_custom_sql(sql)
                if zoneres != None and len(zoneres) != 0:
                    zonename_tag = False
                    for zoneitem in zoneres:
                        zone_name = zoneitem[0]
                        if nameid.find(zone_name) != -1:
                            zonename_tag = True
                        record_name = zoneitem[1]
                        record_content = zoneitem[2]
                        internet_type = zoneitem[3]
                        record_type = zoneitem[4]
                        record_ttl = zoneitem[5]
                        if zoneqdns_dict.get(dnsname) == None:
                            zoneqdns_dict[dnsname] = {}
                        if zoneqdns_dict[dnsname].get(zone_name) == None:
                            zoneqdns_dict[dnsname][zone_name] = []
                        zoneqdns_dict[dnsname][zone_name].append({
                            "record_name":
                            record_name,
                            "record_content":
                            record_content,
                            "internet_type":
                            internet_type,
                            "record_type":
                            record_type,
                            "record_ttl":
                            record_ttl
                        })
                    if len(zoneqdns_dict) == 0 or zonename_tag == False:
                        logger.info(
                            "the nameid zone is not legal,the nameid is {}".
                            format(nameid))
    if len(zoneqdns_dict) == 0:
        logger.info("the zone dict is null")
    keys = get_keys_from_cache("vipdevice", "zone-config")
    for key in keys:
        key = str(key, encoding="raw_unicode_escape")
        if zoneqdns_dict.get(key) == None:
            logger.info("the nameid {} is disable".format(key))
            delete_to_cache_cluster("vipdevice", "zone-config", key)
    for dnstype, zoneinfo in zoneqdns_dict.items():
        logger.info("the dnstype is {},the zone info is {}".format(
            dnstype, json.dumps(zoneinfo)))
        write_to_cache_cluster("vipdevice", "zone-config", str(dnstype),
                               json.dumps(zoneinfo))
def load_device_availability_cache():
    try:
        sql = "select node_isp,total_value,absolute_value,relative_rate from Polaris_tb_fact_detectdeviceavailability_standard_info"
        res = my_custom_sql(sql)
        isp_standard_data = {}
        for item in res:
            isp = item[0]
            total_value = item[1]
            absolute_value = item[2]
            relative_rate = item[3]
            if isp_standard_data.get(isp) == None:
                isp_standard_data[isp] = {}
            isp_standard_data[isp]["total_value"] = total_value
            isp_standard_data[isp]["absolute_value"] = absolute_value
            isp_standard_data[isp]["relative_rate"] = relative_rate

        sql = "select effective_time,effective_time_unit from Polaris_tb_fact_detecttask_info where detect_name='detect_device_availability'"
        res = my_custom_sql(sql)
        effective_time = 15
        effective_time_unit = 'second'
        for item in res:
            effective_time = item[0]
            effective_time_unit = item[1]
            if effective_time == None or effective_time_unit == None:
                effective_time = 15
                effective_time_unit = 'second'
        #这里group by vip和availability,以vip为key,拿到最终上报的关于该vip的总的探针个数,上报的状态,以及运营商
        sql = 'select vip_address,availability,admin_isp,count(availability) from Polaris_tb_fact_detectdeviceavailability_select_info where create_time >= DATE_FORMAT(DATE_ADD(now(), INTERVAL - {} {}),"%y-%m-%d %H:%i:%s") group by vip_address,availability'.format(
            effective_time, effective_time_unit)
        res = my_custom_sql(sql)
        vip_avil_dict = {}
        for item in res:
            vip_address = item[0]
            status = item[1]
            admin_isp = item[2]
            num = item[3]
            #设备的运营商信息以最后一次为准
            if vip_avil_dict.get(vip_address) == None:
                vip_avil_dict[vip_address] = [status, num, admin_isp]
            elif vip_avil_dict[vip_address][1] < num:
                vip_avil_dict[vip_address][0] = status
                vip_avil_dict[vip_address][
                    1] = vip_avil_dict[vip_address][1] + num
                vip_avil_dict[vip_address][2] = admin_isp
            else:
                vip_avil_dict[vip_address][
                    1] = vip_avil_dict[vip_address][1] + num
                vip_avil_dict[vip_address][2] = admin_isp

        detect_vip_status = {}
        #if vip_avil_dict.get(vip_address) == None or vip_avil_dict[vip_address][1] < num:
        #    vip_avil_dict[vip_address] = [status,num,admin_isp]
        #开始确定每个vip的状态,对于每个vip都要满足俩个标准,一个是上报的探针个数要达到那个绝对值,还有一个是要达到那个相对值,会有三种状态,对于数据不符合检验标准的,都认为是无效的,对于无效的探测数据,还是要采用手工配置的方式
        for item in vip_avil_dict:
            status = vip_avil_dict[item][0]
            num = vip_avil_dict[item][1]
            admin_isp = vip_avil_dict[item][2]
            if isp_standard_data.get(admin_isp) == None:
                detect_vip_status[item] = "invalid"
                #write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
            else:
                total_value = isp_standard_data[admin_isp]["total_value"]
                absolute_value = isp_standard_data[admin_isp]["absolute_value"]
                relative_rate = isp_standard_data[admin_isp]["relative_rate"]
                logger.info(
                    "the vip is {},the total detect alive vip num is {},the standared num is {},the relative percent alive is {},the standared is {}"
                    .format(item, num, absolute_value, num / total_value,
                            relative_rate))
                if num < absolute_value or (num / total_value < relative_rate):
                    detect_vip_status[item] = "invalid"
                    #write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,"invalid")
                else:
                    detect_vip_status[item] = vip_avil_dict[item][0]
                    #write_to_cache_cluster("vipdevice","detect-vipaddress-availability",item,vip_avil_dict[item][0])
                logger.info("the vip {},the detect status is {}".format(
                    item,
                    read_from_cache_cluster("vipdevice",
                                            "detect-vipaddress-availability",
                                            item)))
        keys = get_keys_from_cache("vipdevice",
                                   "detect-vipaddress-availability")
        for key in keys:
            key = str(key, encoding="raw_unicode_escape")
            if detect_vip_status.get(key) == None:
                logger.info("the detect vip {} is disable".format(key))
                delete_to_cache_cluster("vipdevice",
                                        "detect-vipaddress-availability", key)
        for vip, status in detect_vip_status.items():
            write_to_cache_cluster("vipdevice",
                                   "detect-vipaddress-availability", vip,
                                   status)
    except Exception as err:
        logger.error(err)
def vipdevice_availability_policy(multiple_config):
    qdns_res = {}
    policy_key = ""
    if multiple_config != "nameid-default" and multiple_config != "nameid-manual":
        return
    objs = tb_fact_nameid_info.objects.all()
    for obj in objs:
        if obj.nameid_name is not None:
            try:
                nameid = obj.nameid_name
                status = obj.nameid_status
                policy = str(obj.nameid_policy)
                policy_key = policy
                dnszone = str(obj.zone_type.zone_name)
                if not nameid.endswith(dnszone):
                    logger.info(
                        "the nameid {} is  not legal...".format(nameid))
                    continue
                nameid_obj = read_from_cache_cluster("vipdevice",
                                                     multiple_config, nameid)
                if nameid_obj != None:
                    nameid_obj = json.loads(nameid_obj)
                    nameid_obj_replace = {}
                    for viewname, viewinfo in nameid_obj.items():
                        if viewinfo != None:
                            nameid_obj_replace[viewname] = {}
                            for k, v in viewinfo.items():
                                if k == "address":
                                    nameid_obj_replace[viewname][k] = {}
                                    for vip, ratio in v.items():
                                        vipaddress_availability = read_from_cache_cluster(
                                            "vipdevice",
                                            "vipaddress-availability", vip)
                                        if vipaddress_availability != None:
                                            if vipaddress_availability == "enable":
                                                nameid_obj_replace[viewname][
                                                    k][vip] = ratio
                                else:
                                    nameid_obj_replace[viewname][k] = v
                            if (
                                    nameid_obj_replace[viewname].get("address")
                                    == None and
                                    nameid_obj_replace[viewname].get("cname")
                                    == None
                            ) or (nameid_obj_replace[viewname].get("address")
                                  != None and len(
                                      nameid_obj_replace[viewname]["address"])
                                  == 0
                                  and nameid_obj_replace[viewname].get("cname")
                                  != None and len(
                                      nameid_obj_replace[viewname]["cname"])
                                  == 0):
                                del nameid_obj_replace[viewname]
                                logger.info(
                                    "the nameid is {},the view {} is null".
                                    format(nameid, viewname))
                    if len(nameid_obj_replace) != 0:
                        if qdns_res.get(nameid) == None:
                            qdns_res[nameid] = {}
                        qdns_res[nameid] = nameid_obj_replace
            except Exception as err:
                logger.error(err)
                continue
    if policy_key == None:
        policy_key = "vipdevice_availability_policy"
    keys = get_keys_from_cache("vipdevice", policy_key)
    for key in keys:
        key = str(key, encoding="raw_unicode_escape")
        if qdns_res.get(key) == None:
            logger.info("the nameid {} is disable".format(key))
            delete_to_cache_cluster("vipdevice", policy_key, key)
    for nameid, nameidinfo in qdns_res.items():
        logger.info("the nameid is {},the nameid info is {}".format(
            nameid, json.dumps(nameidinfo)))
        write_to_cache_cluster("vipdevice", "vipdevice_availability_policy",
                               nameid, json.dumps(nameidinfo))