def Add_Data_To_Url(url):
# close_old_connections()
try:
ip = get_host(url)
if ip == '获取失败':
return
# print('[+ Domain UrlIP] IP解析 --> {} IP --> {}'.format(url, ip))
# Sem.acquire()
try:
test_url = list(URL.objects.filter(url=url))
except:
try:
test_url = list(URL.objects.filter(url=url))
except:
test_url = list(URL.objects.filter(url=url))
# Sem.release()
# 如果数据库有这个网站的话，就直接退出
if test_url != []:
return
try:
Test_Other_Url = Other_Url.objects.filter(url=url)
# 判断网络资产表是否有这个数据，如果没有的话，就添加进去
if list(Test_Other_Url) == []:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
try:
res_title = pymysql.escape_string(res.get('title'))
except Exception as e:
res_title = 'Error'
Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
res_power = res.get('power')
res_server = res.get('server')
res_status = res.get('status')
res_ip = ip
try:
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
Other_Url.objects.create(url=res_url,
title='Error',
power='Error',
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
try:
# res = Get_Url_Info(url).get_info()
# res_status = res.get('status')
# 再次获取状态码，判断是否符合入库状态，以保证数据统一
# if int(res_status) not in Alive_Status:
# return
# 接下来可以进行数据索引唯一统一
'''
这里添加网址资产到 索引表 和 清洗表
'''
test_url1 = list(URL.objects.filter(url=url))
# 如果数据库有这个网站的话，就直接退出
if test_url1 == []:
URL.objects.create(url=url, ip=ip)
# 添加 网址索引
try:
try:
Show_contents = pymysql.escape_string(
Get_Url_Info(url).Requests()[0])
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
except Exception as e:
Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
Show_contents = 'Error'
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
# 添加网页内容，数据展示
except Exception as e:
Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))
This_Sub = [x for x in ALL_DOMAINS if x in url]
# 获取到当前子域名属于的主域名
try:
# 尝试进行域名总数据获取检测
if This_Sub != []:
Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
counts = Other_Url.objects.filter(
url__contains=This_Sub[0])
Domain_Count.counts = str(len(counts))
# counts = int(Domain_Count.counts)+1
# Domain_Count.counts = counts
Domain_Count.save()
except Exception as e:
Except_Log(stat=15,
url=url + '|获取归属域名失败|' + This_Sub,
error=str(e))
except Exception as e:
Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))
test_ip = list(IP.objects.filter(ip=ip))
# 开始添加ip 维护ip统一
# 这里开始判断数据库中是否有这个ip，并且先添加然后修改(防止重复浪费资源)
if test_ip != []:
test_ip_0 = IP.objects.filter(ip=ip)[0]
# 这里判断数据中IP时候存在，如果存在并且有扫描状态，就直接中断操作
if test_ip_0.get == '是' or test_ip_0.get == '中':
return
if test_ip == []:
try:
IP_Res = Get_Ip_Info(ip)
area = IP_Res.get_ip_address(ip)
cs_name = IP_Res.get_cs_name(ip)
try:
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area=area)
# 这里先添加数据，异步执行获取到的数据作为结果给下个进程使用
# 这里本来是要扫描c段开放端口，但是这样就相当于把耗时操作加载到同步执行的线程中
# 于是把扫描开放端口 放在获取ip详细信息线程中处理
except:
Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area='获取失败')
except Exception as e:
Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))
except Exception as e:
Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
def Add_Data_To_Url(url):
'''
2019-12-10
1. 该函数作用为传入网址进行IP黑名单过滤
2. 该函数作用为传入网址进行【网络资产数据入库，网址索引数据入库，主机资产数据入库，监控域名数量入库处理】
2020-01-14
1. 新增泛解析过滤规则
'''
time.sleep(random.randint(5, 20))
time.sleep(random.randint(5, 20))
close_old_connections()
urlhasdomain = check_black(url, ALL_DOMAINS)
if urlhasdomain == False:
print('[+ Insert Url] 当前网址不在域名监控域名范围内 : {}'.format(url))
try:
close_old_connections()
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='当前网址不在域名监控域名范围内')
return
except:
close_old_connections()
return
if '.gov.cn' in url or '.edu.cn' in url:
return
urlinblackurl = check_black(url, black_url)
if urlinblackurl == True:
print('[+ URL Blacklist] 当前网址触发黑名单 : {}'.format(url))
try:
burl = ''
for blacurl in black_url:
if blacurl in url:
burl = blacurl
close_old_connections()
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='触发网址黑名单:{}'.format(burl))
except Exception as e:
pass
return
try:
ip = get_host(url)
if ip == '获取失败':
try:
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='获取网址IP失败')
except Exception as e:
pass
return
if ip in black_ip:
'''触发IP黑名单机制'''
print('[+ IP Blacklist] 当前IP触发黑名单 : {} --> {}'.format(ip, url))
try:
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='触发IP黑名单:{}'.format(ip))
except Exception as e:
pass
return
try:
test_url = list(URL.objects.filter(url=url))
except:
try:
test_url = list(URL.objects.filter(url=url))
except:
close_old_connections()
test_url = list(URL.objects.filter(url=url))
if test_url != []:
'''网址索引表如果已经有该网址，则直接退出'''
return
'''
2020-01-14
1. 这里开始对比泛解析数据，判断是否为泛解析网址
2. 分别获取泛解析对比的 标题，ip，网页内容
3. 然后先对比标题，标题一致，说明不是泛解析哦~不过携程还是哪个大厂名字忘了，访问频率过快的话，网页标题会变成 填写验证码xxxx
4. 其次对比网页内容，如果网页内容相似度过大，则说明泛解析哦~
5. 有人问，为什么不直接对比ip不就行了吗？其实不是的，比如xxadasda.yy.com--->aedqwawrqw668.sdada.yy.com很明显都是泛解析，但是解析的ip都是不一样的
'''
infjx = [x for x in ALL_DOMAINS if x in url]
if infjx == []:
return
else:
infjx = infjx[0]
inftitle, infip, infcontent = DOMAINSINFOS[infjx][
'title'], DOMAINSINFOS[infjx]['ip'], DOMAINSINFOS[infjx]['content']
DD = Get_Url_Info(url).get_info()
comtitle, comip, comcontent = DD['title'], DD['ip'], DD['content']
# if inftitle != comtitle:
# # 如果标题不一样,决策为不是泛解析~,大概是80%的准确率，但是对安居客来说，这一点判断是无效的
# pass
#else:
if infcontent != 'Error' and comcontent != 'Error':
if Return_Content_Difflib(infcontent, comcontent) == True:
try:
print('[+ URL Universal] 泛解析网址自动过滤 : {}'.format(url))
close_old_connections()
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='泛解析自动过滤')
return
except:
return
else:
DD1 = Get_Url_Info(url.replace('://', '://yyyyyyyyy')).get_info()
comtitle1, comip1, comcontent1 = DD1['title'], DD1['ip'], DD1[
'content']
if Return_Content_Difflib(comcontent, comcontent1) == True:
try:
print('[+ URL Universal] 泛解析网址自动过滤 : {}'.format(url))
close_old_connections()
BLACKURL.objects.create(url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='泛解析自动过滤')
return
except:
return
print('[+ Insert Url] 入库网址 : {}'.format(url))
try:
Test_Other_Url = Other_Url.objects.filter(url=url)
'''判断网络资产表是否有这个数据，如果没有的话，就添加进去'''
if list(Test_Other_Url) == []:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
try:
res_title = pymysql.escape_string(res.get('title'))
except Exception as e:
res_title = 'Error'
Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
res_power = res.get('power')
res_server = res.get('server')
res_status = res.get('status')
res_ip = ip
try:
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
close_old_connections()
Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
Other_Url.objects.create(url=res_url,
title='Error',
power='Error',
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
try:
'''
再次获取状态码，判断是否符合入库状态，以保证数据统一
这里添加网址资产到 索引表 和 清洗表
'''
test_url1 = list(URL.objects.filter(url=url))
'''如果网址索引表有这个网站的话，就直接退出'''
if test_url1 == []:
URL.objects.create(url=url, ip=ip)
'''添加网址到网址索引表'''
try:
try:
ZHRND = Get_Url_Info(url)
Sconten = ZHRND.get_info()['content']
if Sconten == 'Error':
'''到这里说明获取网页内容失败了'''
# print('{}:获取网页内容失败'.format(url))
pass
else:
try:
blackconincon = check_black(Sconten, black_con)
if blackconincon == True:
'''触发网页内容黑名单'''
burl = ''
for blacurl in black_con:
if blacurl in Sconten:
burl = blacurl
print(
'[+ Cont Blacklist] 当前网页内容触发黑名单 : {}'.
format(url))
try:
close_old_connections()
BLACKURL.objects.create(
url=url,
ip=get_host(url),
title=RequestsTitle(url),
resons='触发网页内容黑名单:{}'.format(burl))
except Exception as e:
pass
return
except:
Sconten = '获取失败'
Show_contents = pymysql.escape_string(Sconten)
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
except Exception as e:
close_old_connections()
Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
Show_contents = 'Error'
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
'''添加网页内容，数据展示'''
except Exception as e:
Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))
This_Sub = [x for x in ALL_DOMAINS if x in url]
'''获取到当前子域名属于的主域名'''
try:
'''尝试进行域名总数据获取检测'''
if This_Sub != []:
Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
counts = Other_Url.objects.filter(
url__contains=This_Sub[0])
Domain_Count.counts = str(len(counts))
# counts = int(Domain_Count.counts)+1
# Domain_Count.counts = counts
Domain_Count.save()
except Exception as e:
Except_Log(stat=15,
url=url + '|获取归属域名失败|' + This_Sub,
error=str(e))
except Exception as e:
Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))
try:
test_ip = list(IP.objects.filter(ip=ip))
except:
close_old_connections()
test_ip = list(IP.objects.filter(ip=ip))
'''开始添加ip 维护ip统一
这里开始判断数据库中是否有这个ip，并且先添加然后修改(防止重复浪费资源)
if test_ip != []:
test_ip_0 = IP.objects.filter(ip=ip)[0]
# 这里判断数据中IP时候存在，如果存在并且有扫描状态，就直接中断操作
if test_ip_0.get == '是' or test_ip_0.get == '中':
return'''
if test_ip == []:
try:
IP_Res = Get_Ip_Info(ip)
area = IP_Res.get_ip_address(ip)
cs_name = IP_Res.get_cs_name(ip)
try:
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area=area)
'''这里先添加数据，异步执行获取到的数据作为结果给下个进程使用
这里本来是要扫描c段开放端口，但是这样就相当于把耗时操作加载到同步执行的线程中
于是把扫描开放端口 放在获取ip详细信息线程中处理'''
except Exception as e:
Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area='获取失败')
except Exception as e:
Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))
except Exception as e:
Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
def Add_Data_To_Url(url):
try:
ip = get_host(url)
if ip == '获取失败':
return
# print('[+ Domain UrlIP] IP解析 --> {} IP --> {}'.format(url, ip))
test_url = list(URL.objects.filter(url=url))
# 如果数据库有这个网站的话，就直接退出
if test_url != []:
return
try:
Test_Other_Url = Other_Url.objects.filter(url=url)
# 判断网络资产表是否有这个数据，如果没有的话，就添加进去
if list(Test_Other_Url) == []:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
res_title = res.get('title')
res_power = res.get('power')
res_server = res.get('server')
res_status = res.get('status')
res_ip = ip
if int(res_status) in Alive_Status:
# 添加的标准是 在入库状态码内
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
print('错误代码 [29] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [29] {}'.format(str(e)))
try:
res = Get_Url_Info(url).get_info()
res_status = res.get('status')
# 再次获取状态码，判断是否符合入库状态，以保证数据统一
if int(res_status) not in Alive_Status:
return
# 接下来可以进行数据索引唯一统一
URL.objects.create(url=url, ip=ip)
# 添加 网址索引
try:
Show_contents = Get_Url_Info(url).Requests()[0]
Cont = Content()
Cont.url = url
Cont.content = Show_contents
Cont.save()
Show_Data.objects.create(url=url, ip=ip, content=Cont)
# 添加网页内容，数据展示
except Exception as e:
print('错误代码 [08] {}'.format(str(e)))
Error_Log.objects.create(url='外键添加错误',
error='错误代码 [08] {}'.format(str(e)))
BA = Domains.objects.all()
ALL_DOMAINS = [x.get('url') for x in BA.values()]
# 所有监控域名
# print('所有域名：{}'.format(ALL_DOMAINS))
This_Sub = [x for x in ALL_DOMAINS if x in url]
# 获取到当前子域名属于的主域名
try:
# 尝试进行域名总数据获取检测
if This_Sub != []:
Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
counts = Other_Url.objects.filter(
url__contains=This_Sub[0])
Domain_Count.counts = str(len(counts))
# counts = int(Domain_Count.counts)+1
# Domain_Count.counts = counts
Domain_Count.save()
except Exception as e:
print('错误代码 [15] {}'.format(str(e)))
Error_Log.objects.create(url=url + '|' + This_Sub,
error='错误代码 [15] {}'.format(str(e)))
except Exception as e:
print('错误代码 [22] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [22] {}'.format(str(e)))
test_ip = list(IP.objects.filter(ip=ip))
# 开始添加ip 维护ip统一
# 这里开始判断数据库中是否有这个ip，并且先添加然后修改(防止重复浪费资源)
if test_ip != []:
test_ip_0 = IP.objects.filter(ip=ip)[0]
# 这里判断数据中IP时候存在，如果存在并且有扫描状态，就直接中断操作
if test_ip_0.get == '是' or test_ip_0.get == '中':
return
if test_ip == []:
try:
IP_Res = Get_Ip_Info(ip)
area = IP_Res.get_ip_address(ip)
IP.objects.create(ip=ip,
servers='None',
host_type='None',
alive_urls='None',
area=area)
# 这里先添加数据，异步执行获取到的数据作为结果给下个进程使用
except Exception as e:
print('错误代码 [21] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [21] {}'.format(str(e)))
except Exception as e:
print('错误代码 [30] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [30] {}'.format(str(e)))
def Add_Data_To_Url(url):
'''
2019-12-10
1. 该函数作用为传入网址进行IP黑名单过滤
2. 该函数作用为传入网址进行【网络资产数据入库，网址索引数据入库，主机资产数据入库，监控域名数量入库处理】
'''
time.sleep(random.randint(5, 20))
time.sleep(random.randint(5, 20))
time.sleep(random.randint(5, 20))
close_old_connections()
print('[+ Insert Url] 入库网址 : {}'.format(url))
if '.gov.cn' in url or '.edu.cn' in url:
return
urlinblackurl = check_black(url, black_url)
if urlinblackurl == True:
print('[+ URL Blacklist] 当前网址触发黑名单 : {}'.format(url))
return
try:
ip = get_host(url)
if ip == '获取失败':
return
if ip in black_ip:
'''触发IP黑名单机制'''
print('[+ IP Blacklist] 当前IP触发黑名单 : {} --> {}'.format(ip, url))
return
try:
test_url = list(URL.objects.filter(url=url))
except:
try:
test_url = list(URL.objects.filter(url=url))
except:
close_old_connections()
test_url = list(URL.objects.filter(url=url))
if test_url != []:
'''网址索引表如果已经有该网址，则直接退出'''
return
try:
Test_Other_Url = Other_Url.objects.filter(url=url)
'''判断网络资产表是否有这个数据，如果没有的话，就添加进去'''
if list(Test_Other_Url) == []:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
try:
res_title = pymysql.escape_string(res.get('title'))
except Exception as e:
res_title = 'Error'
Except_Log(stat=11, url=url + '|网页内容转码失败', error=str(e))
res_power = res.get('power')
res_server = res.get('server')
res_status = res.get('status')
res_ip = ip
try:
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
close_old_connections()
Except_Log(stat=17, url=url + '|标题等信息编码不符合', error=str(e))
Other_Url.objects.create(url=res_url,
title='Error',
power='Error',
server=res_server,
status=res_status,
ip=res_ip)
except Exception as e:
Except_Log(stat=29, url=url + '|网络资产表错误', error=str(e))
try:
'''
再次获取状态码，判断是否符合入库状态，以保证数据统一
这里添加网址资产到 索引表 和 清洗表
'''
test_url1 = list(URL.objects.filter(url=url))
'''如果网址索引表有这个网站的话，就直接退出'''
if test_url1 == []:
URL.objects.create(url=url, ip=ip)
'''添加网址到网址索引表'''
try:
try:
ZHRND = Get_Url_Info(url)
Sconten = ZHRND.get_info()['content']
if Sconten == 'Error':
'''到这里说明获取网页内容失败了'''
# print('{}:获取网页内容失败'.format(url))
pass
else:
try:
blackconincon = check_black(Sconten, black_con)
if blackconincon == True:
'''触发网页内容黑名单'''
print(
'[+ Cont Blacklist] 当前网页内容触发黑名单 : {}'.
format(url))
return None
except:
Sconten = '获取失败'
Show_contents = pymysql.escape_string(Sconten)
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
except Exception as e:
close_old_connections()
Except_Log(stat=4, url=url + '|外键添加错误', error=str(e))
Show_contents = 'Error'
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url,
ip=ip,
cs=Show_cs,
content=Cont)
'''添加网页内容，数据展示'''
except Exception as e:
Except_Log(stat=8, url=url + '|外键添加错误', error=str(e))
This_Sub = [x for x in ALL_DOMAINS if x in url]
'''获取到当前子域名属于的主域名'''
try:
'''尝试进行域名总数据获取检测'''
if This_Sub != []:
Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
counts = Other_Url.objects.filter(
url__contains=This_Sub[0])
Domain_Count.counts = str(len(counts))
# counts = int(Domain_Count.counts)+1
# Domain_Count.counts = counts
Domain_Count.save()
except Exception as e:
Except_Log(stat=15,
url=url + '|获取归属域名失败|' + This_Sub,
error=str(e))
except Exception as e:
Except_Log(stat=22, url=url + '|添加到网址索引表失败|', error=str(e))
try:
test_ip = list(IP.objects.filter(ip=ip))
except:
close_old_connections()
test_ip = list(IP.objects.filter(ip=ip))
'''开始添加ip 维护ip统一
这里开始判断数据库中是否有这个ip，并且先添加然后修改(防止重复浪费资源)
if test_ip != []:
test_ip_0 = IP.objects.filter(ip=ip)[0]
# 这里判断数据中IP时候存在，如果存在并且有扫描状态，就直接中断操作
if test_ip_0.get == '是' or test_ip_0.get == '中':
return'''
if test_ip == []:
try:
IP_Res = Get_Ip_Info(ip)
area = IP_Res.get_ip_address(ip)
cs_name = IP_Res.get_cs_name(ip)
try:
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area=area)
'''这里先添加数据，异步执行获取到的数据作为结果给下个进程使用
这里本来是要扫描c段开放端口，但是这样就相当于把耗时操作加载到同步执行的线程中
于是把扫描开放端口 放在获取ip详细信息线程中处理'''
except Exception as e:
Except_Log(stat=86, url=url + '|转换IP地区编码失败|', error=str(e))
IP.objects.create(ip=ip,
servers='None',
host_type='None',
cs=cs_name,
alive_urls='None',
area='获取失败')
except Exception as e:
Except_Log(stat=21, url=url + '|添加IP资源失败|', error=str(e))
except Exception as e:
Except_Log(stat=30, url=url + '|维护传入网址失败|', error=str(e))
Add_Data_To_Url(url)
def Add_Data_To_Url(url):
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
time.sleep(random.randint(1,20))
try:
ip = get_host(url)
if ip == '获取失败':
return
# print('[+ Domain UrlIP] IP解析 --> {} IP --> {}'.format(url, ip))
test_url = list(URL.objects.filter(url=url))
# 如果数据库有这个网站的话，就直接退出
if test_url != []:
return
try:
Test_Other_Url = Other_Url.objects.filter(url=url)
# 判断网络资产表是否有这个数据，如果没有的话，就添加进去
if list(Test_Other_Url) == []:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
res_title = res.get('title')
res_power = res.get('power')
res_server = res.get('server')
res_status = res.get('status')
res_ip = ip
#if int(res_status) in Alive_Status:
# 添加的标准是 在入库状态码内
Other_Url.objects.create(url=res_url, title=res_title, power=res_power, server=res_server,
status=res_status,ip=res_ip)
except Exception as e:
print('错误代码 [29] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [29] {}'.format(str(e)))
try:
# res = Get_Url_Info(url).get_info()
# res_status = res.get('status')
# 再次获取状态码，判断是否符合入库状态，以保证数据统一
# if int(res_status) not in Alive_Status:
# return
# 接下来可以进行数据索引唯一统一
test_url1 = list(URL.objects.filter(url=url))
# 如果数据库有这个网站的话，就直接退出
if test_url1 == []:
URL.objects.create(url=url,ip=ip)
# 添加 网址索引
try:
Show_contents = Get_Url_Info(url).Requests()[0]
Cont = Content()
Cont.url = url
Cont.content = Show_contents
IP_Res = Get_Ip_Info(ip)
Show_cs = IP_Res.get_cs_name(ip)
Cont.save()
Show_Data.objects.create(url=url, ip=ip,cs=Show_cs, content=Cont)
# 添加网页内容，数据展示
except Exception as e:
print('错误代码 [08] {}'.format(str(e)))
Error_Log.objects.create(url='外键添加错误', error='错误代码 [08] {}'.format(str(e)))
BA = Domains.objects.all()
ALL_DOMAINS = [x.get('url') for x in BA.values()]
# 所有监控域名
# print('所有域名：{}'.format(ALL_DOMAINS))
This_Sub = [x for x in ALL_DOMAINS if x in url]
# 获取到当前子域名属于的主域名
try:
# 尝试进行域名总数据获取检测
if This_Sub != []:
Domain_Count = Domains.objects.filter(url=This_Sub[0])[0]
counts = Other_Url.objects.filter(url__contains=This_Sub[0])
Domain_Count.counts = str(len(counts))
# counts = int(Domain_Count.counts)+1
# Domain_Count.counts = counts
Domain_Count.save()
except Exception as e:
print('错误代码 [15] {}'.format(str(e)))
Error_Log.objects.create(url=url+'|'+This_Sub, error='错误代码 [15] {}'.format(str(e)))
except Exception as e:
print('错误代码 [22] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [22] {}'.format(str(e)))
test_ip = list(IP.objects.filter(ip=ip))
# 开始添加ip 维护ip统一
# 这里开始判断数据库中是否有这个ip，并且先添加然后修改(防止重复浪费资源)
if test_ip != []:
test_ip_0 = IP.objects.filter(ip=ip)[0]
# 这里判断数据中IP时候存在，如果存在并且有扫描状态，就直接中断操作
if test_ip_0.get == '是' or test_ip_0.get == '中':
return
if test_ip ==[]:
try:
IP_Res = Get_Ip_Info(ip)
area = IP_Res.get_ip_address(ip)
cs_name = IP_Res.get_cs_name(ip)
IP.objects.create(ip=ip, servers='None', host_type='None', cs=cs_name,alive_urls='None', area=area)
# 这里先添加数据，异步执行获取到的数据作为结果给下个进程使用
cs_ips = [str(x) for x in list(IP_Res.get_cs_ips(ip).values())[0]]
# 整个 C 段的数据ip
if ip in cs_ips:
cs_ips.remove(ip)
Read_to_check_host = set()
for cs_ip in cs_ips:
indata = list(IP.objects.filter(ip=str(cs_ip)))
if indata== [] and cs_ip != ip:
Read_to_check_host.add(cs_ip)
Alive_Hosts = IP_Res.get_alive_hosts(Read_to_check_host)
print('[+ CHost Scaner] {} 段存活主机 : {}台'.format(cs_name,len(Alive_Hosts)))
if Alive_Hosts == []:
return
for alive_host in Alive_Hosts:
try:
checkindata = list(IP.objects.filter(ip=str(alive_host)))
if checkindata == [] :
# 最后一次数据判断匹配
c_ip = str(alive_host)
c_cs = cs_name
c_area = IP_Res.get_ip_address(c_ip)
IP.objects.create(ip=c_ip, servers='None', host_type='None', cs=c_cs, alive_urls='None',
area=c_area)
except Exception as e:
print('错误代码 [03] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [03] {}'.format(str(e)))
except Exception as e:
print('错误代码 [21] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [21] {}'.format(str(e)))
except Exception as e:
print('错误代码 [30] {}'.format(str(e)))
Error_Log.objects.create(url=url, error='错误代码 [30] {}'.format(str(e)))
