示例#1
0
def Add_Data_To_Url(url):
    # close_old_connections()
    try:
        ip = get_host(url)
        if ip == '获取失败':
            return
        # print('[+ Domain UrlIP] IP解析 --> {}  IP --> {}'.format(url, ip))
    #  Sem.acquire()
        try:
            test_url = list(URL.objects.filter(url=url))
        except:
            try:
                test_url = list(URL.objects.filter(url=url))
            except:
                test_url = list(URL.objects.filter(url=url))

        # Sem.release()
        # 如果数据库有这个网站的话,就直接退出
        if test_url != []:
            return

        try:
            Test_Other_Url = Other_Url.objects.filter(url=url)
            # 判断网络资产表是否有这个数据,如果没有的话,就添加进去
            if list(Test_Other_Url) == []:
                res = Get_Url_Info(url).get_info()
                res_url = res.get('url')
                try:
                    res_title = pymysql.escape_string(res.get('title'))
                except Exception as e:
                    res_title = 'Error'
                    Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
                res_power = res.get('power')
                res_server = res.get('server')
                res_status = res.get('status')
                res_ip = ip
                try:
                    Other_Url.objects.create(url=res_url,
                                             title=res_title,
                                             power=res_power,
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
                except Exception as e:
                    Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
                    Other_Url.objects.create(url=res_url,
                                             title='Error',
                                             power='Error',
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
        except Exception as e:
            Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
        try:
            # res = Get_Url_Info(url).get_info()
            # res_status = res.get('status')
            # 再次获取状态码,判断是否符合入库状态,以保证数据统一
            # if int(res_status) not in Alive_Status:
            #     return

            # 接下来可以进行数据索引唯一统一
            '''
            这里添加网址资产到 索引表 和 清洗表
            '''
            test_url1 = list(URL.objects.filter(url=url))
            # 如果数据库有这个网站的话,就直接退出

            if test_url1 == []:
                URL.objects.create(url=url, ip=ip)
                # 添加 网址索引
                try:
                    try:
                        Show_contents = pymysql.escape_string(
                            Get_Url_Info(url).Requests()[0])
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                    except Exception as e:
                        Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
                        Show_contents = 'Error'
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                    # 添加网页内容,数据展示
                except Exception as e:
                    Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))

                This_Sub = [x for x in ALL_DOMAINS if x in url]
                # 获取到当前子域名属于的主域名
            try:
                # 尝试进行域名总数据获取检测
                if This_Sub != []:
                    Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
                    counts = Other_Url.objects.filter(
                        url__contains=This_Sub[0])
                    Domain_Count.counts = str(len(counts))
                    # counts = int(Domain_Count.counts)+1
                    # Domain_Count.counts = counts
                    Domain_Count.save()
            except Exception as e:
                Except_Log(stat=15,
                           url=url + '|获取归属域名失败|' + This_Sub,
                           error=str(e))
        except Exception as e:
            Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))

        test_ip = list(IP.objects.filter(ip=ip))
        # 开始添加ip 维护ip统一
        # 这里开始判断数据库中是否有这个ip,并且先添加然后修改(防止重复浪费资源)
        if test_ip != []:
            test_ip_0 = IP.objects.filter(ip=ip)[0]
            # 这里判断数据中IP时候存在,如果存在并且有扫描状态,就直接中断操作
            if test_ip_0.get == '是' or test_ip_0.get == '中':
                return
        if test_ip == []:
            try:
                IP_Res = Get_Ip_Info(ip)
                area = IP_Res.get_ip_address(ip)
                cs_name = IP_Res.get_cs_name(ip)
                try:
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area=area)
                    # 这里先添加数据,异步执行获取到的数据作为结果给下个进程使用
                    # 这里本来是要扫描c段开放端口,但是这样就相当于把耗时操作加载到同步执行的线程中
                    # 于是把扫描开放端口  放在获取ip详细信息线程中处理
                except:
                    Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area='获取失败')

            except Exception as e:
                Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))

    except Exception as e:
        Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
示例#2
0
def Add_Data_To_Url(url):
    '''
    2019-12-10
        1. 该函数作用为传入网址进行IP黑名单过滤
        2. 该函数作用为传入网址进行【网络资产数据入库,网址索引数据入库,主机资产数据入库,监控域名数量入库处理】
    2020-01-14
        1. 新增泛解析过滤规则
    '''
    time.sleep(random.randint(5, 20))
    time.sleep(random.randint(5, 20))
    close_old_connections()
    urlhasdomain = check_black(url, ALL_DOMAINS)
    if urlhasdomain == False:
        print('[+ Insert Url] 当前网址不在域名监控域名范围内 : {}'.format(url))
        try:
            close_old_connections()
            BLACKURL.objects.create(url=url,
                                    ip=get_host(url),
                                    title=RequestsTitle(url),
                                    resons='当前网址不在域名监控域名范围内')
            return
        except:
            close_old_connections()
            return
    if '.gov.cn' in url or '.edu.cn' in url:
        return
    urlinblackurl = check_black(url, black_url)
    if urlinblackurl == True:
        print('[+ URL Blacklist] 当前网址触发黑名单 : {}'.format(url))
        try:
            burl = ''
            for blacurl in black_url:
                if blacurl in url:
                    burl = blacurl
            close_old_connections()
            BLACKURL.objects.create(url=url,
                                    ip=get_host(url),
                                    title=RequestsTitle(url),
                                    resons='触发网址黑名单:{}'.format(burl))
        except Exception as e:
            pass
        return

    try:
        ip = get_host(url)
        if ip == '获取失败':
            try:
                BLACKURL.objects.create(url=url,
                                        ip=get_host(url),
                                        title=RequestsTitle(url),
                                        resons='获取网址IP失败')
            except Exception as e:
                pass
            return
        if ip in black_ip:
            '''触发IP黑名单机制'''
            print('[+ IP Blacklist] 当前IP触发黑名单 : {} --> {}'.format(ip, url))
            try:
                BLACKURL.objects.create(url=url,
                                        ip=get_host(url),
                                        title=RequestsTitle(url),
                                        resons='触发IP黑名单:{}'.format(ip))
            except Exception as e:
                pass
            return

        try:
            test_url = list(URL.objects.filter(url=url))
        except:
            try:
                test_url = list(URL.objects.filter(url=url))
            except:
                close_old_connections()
                test_url = list(URL.objects.filter(url=url))

        if test_url != []:
            '''网址索引表如果已经有该网址,则直接退出'''
            return
        '''
        2020-01-14
        1. 这里开始对比泛解析数据,判断是否为泛解析网址
        2. 分别获取泛解析对比的 标题,ip,网页内容
        3. 然后先对比标题,标题一致,说明不是泛解析哦~不过携程还是哪个大厂名字忘了,访问频率过快的话,网页标题会变成 填写验证码xxxx
        4. 其次对比网页内容,如果网页内容相似度过大,则说明泛解析哦~
        5. 有人问,为什么不直接对比ip不就行了吗?其实不是的,比如xxadasda.yy.com--->aedqwawrqw668.sdada.yy.com很明显都是泛解析,但是解析的ip都是不一样的
        '''
        infjx = [x for x in ALL_DOMAINS if x in url]
        if infjx == []:
            return
        else:
            infjx = infjx[0]
        inftitle, infip, infcontent = DOMAINSINFOS[infjx][
            'title'], DOMAINSINFOS[infjx]['ip'], DOMAINSINFOS[infjx]['content']
        DD = Get_Url_Info(url).get_info()
        comtitle, comip, comcontent = DD['title'], DD['ip'], DD['content']
        # if inftitle != comtitle:
        #     # 如果标题不一样,决策为不是泛解析~,大概是80%的准确率,但是对安居客来说,这一点判断是无效的
        #     pass
        #else:
        if infcontent != 'Error' and comcontent != 'Error':
            if Return_Content_Difflib(infcontent, comcontent) == True:
                try:
                    print('[+ URL Universal] 泛解析网址自动过滤 : {}'.format(url))
                    close_old_connections()
                    BLACKURL.objects.create(url=url,
                                            ip=get_host(url),
                                            title=RequestsTitle(url),
                                            resons='泛解析自动过滤')
                    return
                except:
                    return
        else:
            DD1 = Get_Url_Info(url.replace('://', '://yyyyyyyyy')).get_info()
            comtitle1, comip1, comcontent1 = DD1['title'], DD1['ip'], DD1[
                'content']
            if Return_Content_Difflib(comcontent, comcontent1) == True:
                try:
                    print('[+ URL Universal] 泛解析网址自动过滤 : {}'.format(url))
                    close_old_connections()
                    BLACKURL.objects.create(url=url,
                                            ip=get_host(url),
                                            title=RequestsTitle(url),
                                            resons='泛解析自动过滤')
                    return
                except:
                    return

        print('[+ Insert Url] 入库网址 : {}'.format(url))
        try:
            Test_Other_Url = Other_Url.objects.filter(url=url)
            '''判断网络资产表是否有这个数据,如果没有的话,就添加进去'''
            if list(Test_Other_Url) == []:
                res = Get_Url_Info(url).get_info()
                res_url = res.get('url')
                try:
                    res_title = pymysql.escape_string(res.get('title'))
                except Exception as e:
                    res_title = 'Error'
                    Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
                res_power = res.get('power')
                res_server = res.get('server')
                res_status = res.get('status')
                res_ip = ip
                try:
                    Other_Url.objects.create(url=res_url,
                                             title=res_title,
                                             power=res_power,
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
                except Exception as e:
                    close_old_connections()
                    Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
                    Other_Url.objects.create(url=res_url,
                                             title='Error',
                                             power='Error',
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
        except Exception as e:
            Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
        try:
            '''
            再次获取状态码,判断是否符合入库状态,以保证数据统一
            这里添加网址资产到 索引表 和 清洗表
            '''
            test_url1 = list(URL.objects.filter(url=url))
            '''如果网址索引表有这个网站的话,就直接退出'''
            if test_url1 == []:
                URL.objects.create(url=url, ip=ip)
                '''添加网址到网址索引表'''
                try:
                    try:
                        ZHRND = Get_Url_Info(url)
                        Sconten = ZHRND.get_info()['content']
                        if Sconten == 'Error':
                            '''到这里说明获取网页内容失败了'''
                            # print('{}:获取网页内容失败'.format(url))
                            pass
                        else:
                            try:
                                blackconincon = check_black(Sconten, black_con)
                                if blackconincon == True:
                                    '''触发网页内容黑名单'''
                                    burl = ''
                                    for blacurl in black_con:
                                        if blacurl in Sconten:
                                            burl = blacurl
                                    print(
                                        '[+ Cont Blacklist] 当前网页内容触发黑名单 : {}'.
                                        format(url))
                                    try:
                                        close_old_connections()
                                        BLACKURL.objects.create(
                                            url=url,
                                            ip=get_host(url),
                                            title=RequestsTitle(url),
                                            resons='触发网页内容黑名单:{}'.format(burl))
                                    except Exception as e:
                                        pass
                                    return
                            except:
                                Sconten = '获取失败'
                        Show_contents = pymysql.escape_string(Sconten)
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                    except Exception as e:
                        close_old_connections()
                        Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
                        Show_contents = 'Error'
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                        '''添加网页内容,数据展示'''
                except Exception as e:
                    Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))

            This_Sub = [x for x in ALL_DOMAINS if x in url]
            '''获取到当前子域名属于的主域名'''
            try:
                '''尝试进行域名总数据获取检测'''
                if This_Sub != []:
                    Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
                    counts = Other_Url.objects.filter(
                        url__contains=This_Sub[0])
                    Domain_Count.counts = str(len(counts))
                    # counts = int(Domain_Count.counts)+1
                    # Domain_Count.counts = counts
                    Domain_Count.save()
            except Exception as e:
                Except_Log(stat=15,
                           url=url + '|获取归属域名失败|' + This_Sub,
                           error=str(e))
        except Exception as e:
            Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))

        try:
            test_ip = list(IP.objects.filter(ip=ip))
        except:
            close_old_connections()
            test_ip = list(IP.objects.filter(ip=ip))
            '''开始添加ip 维护ip统一
            这里开始判断数据库中是否有这个ip,并且先添加然后修改(防止重复浪费资源)
            if test_ip != []:
                test_ip_0 = IP.objects.filter(ip=ip)[0]
                # 这里判断数据中IP时候存在,如果存在并且有扫描状态,就直接中断操作
                if test_ip_0.get == '是' or test_ip_0.get == '中':
                    return'''
        if test_ip == []:
            try:
                IP_Res = Get_Ip_Info(ip)
                area = IP_Res.get_ip_address(ip)
                cs_name = IP_Res.get_cs_name(ip)
                try:
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area=area)
                    '''这里先添加数据,异步执行获取到的数据作为结果给下个进程使用
                    这里本来是要扫描c段开放端口,但是这样就相当于把耗时操作加载到同步执行的线程中
                    于是把扫描开放端口  放在获取ip详细信息线程中处理'''
                except Exception as e:
                    Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area='获取失败')

            except Exception as e:
                Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))

    except Exception as e:
        Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
示例#3
0
def Add_Data_To_Url(url):
    try:
        ip = get_host(url)
        if ip == '获取失败':
            return
        # print('[+ Domain UrlIP] IP解析 --> {}  IP --> {}'.format(url, ip))
        test_url = list(URL.objects.filter(url=url))
        # 如果数据库有这个网站的话,就直接退出
        if test_url != []:
            return

        try:
            Test_Other_Url = Other_Url.objects.filter(url=url)
            # 判断网络资产表是否有这个数据,如果没有的话,就添加进去
            if list(Test_Other_Url) == []:
                res = Get_Url_Info(url).get_info()
                res_url = res.get('url')
                res_title = res.get('title')
                res_power = res.get('power')
                res_server = res.get('server')
                res_status = res.get('status')
                res_ip = ip
                if int(res_status) in Alive_Status:
                    # 添加的标准是 在入库状态码内
                    Other_Url.objects.create(url=res_url,
                                             title=res_title,
                                             power=res_power,
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
        except Exception as e:
            print('错误代码 [29] {}'.format(str(e)))
            Error_Log.objects.create(url=url,
                                     error='错误代码 [29] {}'.format(str(e)))

        try:
            res = Get_Url_Info(url).get_info()
            res_status = res.get('status')
            # 再次获取状态码,判断是否符合入库状态,以保证数据统一
            if int(res_status) not in Alive_Status:
                return

            # 接下来可以进行数据索引唯一统一

            URL.objects.create(url=url, ip=ip)
            # 添加 网址索引
            try:
                Show_contents = Get_Url_Info(url).Requests()[0]
                Cont = Content()
                Cont.url = url
                Cont.content = Show_contents
                Cont.save()
                Show_Data.objects.create(url=url, ip=ip, content=Cont)
                # 添加网页内容,数据展示
            except Exception as e:
                print('错误代码 [08] {}'.format(str(e)))
                Error_Log.objects.create(url='外键添加错误',
                                         error='错误代码 [08] {}'.format(str(e)))

            BA = Domains.objects.all()
            ALL_DOMAINS = [x.get('url') for x in BA.values()]
            # 所有监控域名
            # print('所有域名:{}'.format(ALL_DOMAINS))
            This_Sub = [x for x in ALL_DOMAINS if x in url]
            # 获取到当前子域名属于的主域名

            try:
                # 尝试进行域名总数据获取检测
                if This_Sub != []:
                    Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
                    counts = Other_Url.objects.filter(
                        url__contains=This_Sub[0])
                    Domain_Count.counts = str(len(counts))
                    # counts = int(Domain_Count.counts)+1
                    # Domain_Count.counts = counts
                    Domain_Count.save()
            except Exception as e:
                print('错误代码 [15] {}'.format(str(e)))
                Error_Log.objects.create(url=url + '|' + This_Sub,
                                         error='错误代码 [15] {}'.format(str(e)))
        except Exception as e:
            print('错误代码 [22] {}'.format(str(e)))
            Error_Log.objects.create(url=url,
                                     error='错误代码 [22] {}'.format(str(e)))

        test_ip = list(IP.objects.filter(ip=ip))
        # 开始添加ip 维护ip统一
        # 这里开始判断数据库中是否有这个ip,并且先添加然后修改(防止重复浪费资源)
        if test_ip != []:
            test_ip_0 = IP.objects.filter(ip=ip)[0]
            # 这里判断数据中IP时候存在,如果存在并且有扫描状态,就直接中断操作
            if test_ip_0.get == '是' or test_ip_0.get == '中':
                return
        if test_ip == []:
            try:
                IP_Res = Get_Ip_Info(ip)
                area = IP_Res.get_ip_address(ip)
                IP.objects.create(ip=ip,
                                  servers='None',
                                  host_type='None',
                                  alive_urls='None',
                                  area=area)
                # 这里先添加数据,异步执行获取到的数据作为结果给下个进程使用
            except Exception as e:
                print('错误代码 [21] {}'.format(str(e)))
                Error_Log.objects.create(url=url,
                                         error='错误代码 [21] {}'.format(str(e)))
    except Exception as e:
        print('错误代码 [30] {}'.format(str(e)))
        Error_Log.objects.create(url=url, error='错误代码 [30] {}'.format(str(e)))
示例#4
0
def Add_Data_To_Url(url):
    '''
    2019-12-10
        1. 该函数作用为传入网址进行IP黑名单过滤
        2. 该函数作用为传入网址进行【网络资产数据入库,网址索引数据入库,主机资产数据入库,监控域名数量入库处理】
    '''
    time.sleep(random.randint(5, 20))
    time.sleep(random.randint(5, 20))
    time.sleep(random.randint(5, 20))
    close_old_connections()
    print('[+ Insert Url] 入库网址 : {}'.format(url))
    if '.gov.cn' in url or '.edu.cn' in url:
        return
    urlinblackurl = check_black(url, black_url)
    if urlinblackurl == True:
        print('[+ URL Blacklist] 当前网址触发黑名单 : {}'.format(url))
        return
    try:
        ip = get_host(url)
        if ip == '获取失败':
            return
        if ip in black_ip:
            '''触发IP黑名单机制'''
            print('[+ IP Blacklist] 当前IP触发黑名单 : {} --> {}'.format(ip, url))
            return
        try:
            test_url = list(URL.objects.filter(url=url))
        except:
            try:
                test_url = list(URL.objects.filter(url=url))
            except:
                close_old_connections()
                test_url = list(URL.objects.filter(url=url))

        if test_url != []:
            '''网址索引表如果已经有该网址,则直接退出'''
            return

        try:
            Test_Other_Url = Other_Url.objects.filter(url=url)
            '''判断网络资产表是否有这个数据,如果没有的话,就添加进去'''
            if list(Test_Other_Url) == []:
                res = Get_Url_Info(url).get_info()
                res_url = res.get('url')
                try:
                    res_title = pymysql.escape_string(res.get('title'))
                except Exception as e:
                    res_title = 'Error'
                    Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
                res_power = res.get('power')
                res_server = res.get('server')
                res_status = res.get('status')
                res_ip = ip
                try:
                    Other_Url.objects.create(url=res_url,
                                             title=res_title,
                                             power=res_power,
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
                except Exception as e:
                    close_old_connections()
                    Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
                    Other_Url.objects.create(url=res_url,
                                             title='Error',
                                             power='Error',
                                             server=res_server,
                                             status=res_status,
                                             ip=res_ip)
        except Exception as e:
            Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
        try:
            '''
            再次获取状态码,判断是否符合入库状态,以保证数据统一
            这里添加网址资产到 索引表 和 清洗表
            '''
            test_url1 = list(URL.objects.filter(url=url))
            '''如果网址索引表有这个网站的话,就直接退出'''
            if test_url1 == []:
                URL.objects.create(url=url, ip=ip)
                '''添加网址到网址索引表'''
                try:
                    try:
                        ZHRND = Get_Url_Info(url)
                        Sconten = ZHRND.get_info()['content']
                        if Sconten == 'Error':
                            '''到这里说明获取网页内容失败了'''
                            # print('{}:获取网页内容失败'.format(url))
                            pass
                        else:
                            try:
                                blackconincon = check_black(Sconten, black_con)
                                if blackconincon == True:
                                    '''触发网页内容黑名单'''
                                    print(
                                        '[+ Cont Blacklist] 当前网页内容触发黑名单 : {}'.
                                        format(url))
                                    return None
                            except:
                                Sconten = '获取失败'
                        Show_contents = pymysql.escape_string(Sconten)
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                    except Exception as e:
                        close_old_connections()
                        Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
                        Show_contents = 'Error'
                        Cont = Content()
                        Cont.url = url
                        Cont.content = Show_contents
                        IP_Res = Get_Ip_Info(ip)
                        Show_cs = IP_Res.get_cs_name(ip)
                        Cont.save()
                        Show_Data.objects.create(url=url,
                                                 ip=ip,
                                                 cs=Show_cs,
                                                 content=Cont)
                        '''添加网页内容,数据展示'''
                except Exception as e:
                    Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))

            This_Sub = [x for x in ALL_DOMAINS if x in url]
            '''获取到当前子域名属于的主域名'''
            try:
                '''尝试进行域名总数据获取检测'''
                if This_Sub != []:
                    Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
                    counts = Other_Url.objects.filter(
                        url__contains=This_Sub[0])
                    Domain_Count.counts = str(len(counts))
                    # counts = int(Domain_Count.counts)+1
                    # Domain_Count.counts = counts
                    Domain_Count.save()
            except Exception as e:
                Except_Log(stat=15,
                           url=url + '|获取归属域名失败|' + This_Sub,
                           error=str(e))
        except Exception as e:
            Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))

        try:
            test_ip = list(IP.objects.filter(ip=ip))
        except:
            close_old_connections()
            test_ip = list(IP.objects.filter(ip=ip))
            '''开始添加ip 维护ip统一
            这里开始判断数据库中是否有这个ip,并且先添加然后修改(防止重复浪费资源)
            if test_ip != []:
                test_ip_0 = IP.objects.filter(ip=ip)[0]
                # 这里判断数据中IP时候存在,如果存在并且有扫描状态,就直接中断操作
                if test_ip_0.get == '是' or test_ip_0.get == '中':
                    return'''
        if test_ip == []:
            try:
                IP_Res = Get_Ip_Info(ip)
                area = IP_Res.get_ip_address(ip)
                cs_name = IP_Res.get_cs_name(ip)
                try:
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area=area)
                    '''这里先添加数据,异步执行获取到的数据作为结果给下个进程使用
                    这里本来是要扫描c段开放端口,但是这样就相当于把耗时操作加载到同步执行的线程中
                    于是把扫描开放端口  放在获取ip详细信息线程中处理'''
                except Exception as e:
                    Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
                    IP.objects.create(ip=ip,
                                      servers='None',
                                      host_type='None',
                                      cs=cs_name,
                                      alive_urls='None',
                                      area='获取失败')

            except Exception as e:
                Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))

    except Exception as e:
        Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
        Add_Data_To_Url(url)
示例#5
0
def Add_Data_To_Url(url):
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    time.sleep(random.randint(1,20))
    try:
        ip = get_host(url)
        if ip == '获取失败':
            return
        # print('[+ Domain UrlIP] IP解析 --> {}  IP --> {}'.format(url, ip))
        test_url = list(URL.objects.filter(url=url))
        # 如果数据库有这个网站的话,就直接退出
        if test_url != []:
            return

        try:
            Test_Other_Url = Other_Url.objects.filter(url=url)
            # 判断网络资产表是否有这个数据,如果没有的话,就添加进去
            if list(Test_Other_Url) == []:
                res = Get_Url_Info(url).get_info()
                res_url = res.get('url')
                res_title = res.get('title')
                res_power = res.get('power')
                res_server = res.get('server')
                res_status = res.get('status')
                res_ip = ip
                #if int(res_status) in Alive_Status:
                    # 添加的标准是 在入库状态码内
                Other_Url.objects.create(url=res_url, title=res_title, power=res_power, server=res_server,
                                             status=res_status,ip=res_ip)

        except Exception as e:
            print('错误代码 [29] {}'.format(str(e)))
            Error_Log.objects.create(url=url, error='错误代码 [29] {}'.format(str(e)))

        try:
            # res = Get_Url_Info(url).get_info()
            # res_status = res.get('status')
            # 再次获取状态码,判断是否符合入库状态,以保证数据统一
            # if int(res_status) not in Alive_Status:
            #     return

            # 接下来可以进行数据索引唯一统一
            test_url1 = list(URL.objects.filter(url=url))
            # 如果数据库有这个网站的话,就直接退出

            if test_url1 == []:
                URL.objects.create(url=url,ip=ip)
                # 添加 网址索引
                try:
                    Show_contents = Get_Url_Info(url).Requests()[0]
                    Cont = Content()
                    Cont.url = url
                    Cont.content = Show_contents
                    IP_Res = Get_Ip_Info(ip)
                    Show_cs = IP_Res.get_cs_name(ip)
                    Cont.save()
                    Show_Data.objects.create(url=url, ip=ip,cs=Show_cs, content=Cont)
                    # 添加网页内容,数据展示
                except Exception as e:
                    print('错误代码 [08] {}'.format(str(e)))
                    Error_Log.objects.create(url='外键添加错误', error='错误代码 [08] {}'.format(str(e)))

            BA = Domains.objects.all()
            ALL_DOMAINS = [x.get('url') for x in BA.values()]
            # 所有监控域名
            # print('所有域名:{}'.format(ALL_DOMAINS))
            This_Sub = [x for x in ALL_DOMAINS if x in url]
            # 获取到当前子域名属于的主域名

            try:
                # 尝试进行域名总数据获取检测
                if This_Sub != []:
                    Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
                    counts = Other_Url.objects.filter(url__contains=This_Sub[0])
                    Domain_Count.counts = str(len(counts))
                    # counts = int(Domain_Count.counts)+1
                    # Domain_Count.counts = counts
                    Domain_Count.save()
            except Exception as e:
                print('错误代码 [15] {}'.format(str(e)))
                Error_Log.objects.create(url=url+'|'+This_Sub, error='错误代码 [15] {}'.format(str(e)))
        except Exception as e:
            print('错误代码 [22] {}'.format(str(e)))
            Error_Log.objects.create(url=url, error='错误代码 [22] {}'.format(str(e)))

        test_ip = list(IP.objects.filter(ip=ip))
        # 开始添加ip 维护ip统一
        # 这里开始判断数据库中是否有这个ip,并且先添加然后修改(防止重复浪费资源)
        if test_ip != []:
            test_ip_0 = IP.objects.filter(ip=ip)[0]
            # 这里判断数据中IP时候存在,如果存在并且有扫描状态,就直接中断操作
            if test_ip_0.get == '是' or test_ip_0.get == '中':
                return
        if test_ip ==[]:
            try:
                IP_Res = Get_Ip_Info(ip)
                area = IP_Res.get_ip_address(ip)
                cs_name = IP_Res.get_cs_name(ip)
                IP.objects.create(ip=ip, servers='None', host_type='None', cs=cs_name,alive_urls='None', area=area)
                # 这里先添加数据,异步执行获取到的数据作为结果给下个进程使用

                cs_ips = [str(x) for x in list(IP_Res.get_cs_ips(ip).values())[0]]
                # 整个 C 段的数据ip
                if ip in cs_ips:
                    cs_ips.remove(ip)

                Read_to_check_host = set()
                for cs_ip in cs_ips:
                    indata = list(IP.objects.filter(ip=str(cs_ip)))
                    if indata== [] and cs_ip != ip:
                        Read_to_check_host.add(cs_ip)

                Alive_Hosts = IP_Res.get_alive_hosts(Read_to_check_host)
                print('[+ CHost Scaner] {} 段存活主机 : {}台'.format(cs_name,len(Alive_Hosts)))
                if Alive_Hosts == []:
                    return
                for alive_host in Alive_Hosts:
                    try:
                        checkindata = list(IP.objects.filter(ip=str(alive_host)))
                        if checkindata == [] :
                            # 最后一次数据判断匹配
                            c_ip = str(alive_host)
                            c_cs = cs_name
                            c_area = IP_Res.get_ip_address(c_ip)
                            IP.objects.create(ip=c_ip, servers='None', host_type='None', cs=c_cs, alive_urls='None',
                                              area=c_area)
                    except Exception as e:
                        print('错误代码 [03] {}'.format(str(e)))
                        Error_Log.objects.create(url=url, error='错误代码 [03] {}'.format(str(e)))

            except Exception as e:
                print('错误代码 [21] {}'.format(str(e)))
                Error_Log.objects.create(url=url, error='错误代码 [21] {}'.format(str(e)))
    except Exception as e:
        print('错误代码 [30] {}'.format(str(e)))
        Error_Log.objects.create(url=url, error='错误代码 [30] {}'.format(str(e)))