def save_info(self, ips, source, crawl_time):
start = time.time()
all_count = len(ips)
avail_count = 0
if len(ips) > 0:
try:
for ip, updatetime, source in ips:
flag = db.session.query(IP).filter(
IP.ip == ip, IP.source == source).first()
if flag is None:
new_ip = IP()
new_ip.ip = ip
new_ip.updatetime = updatetime
new_ip.source = source
db.session.add(new_ip)
avail_count += 1
else:
flag.updatetime = updatetime
db.session.add(flag)
db.session.commit()
except Exception as e:
self.logger.warning("Error writing to database" + str(e) +
source)
else:
self.logger.warning("NO record found from: %s" % source)
stop = time.time()
storage_time = str(stop - start) + "秒"
self.logger.info("rulez 共收集{0}条数据， 新数据{1}条".format(
all_count, avail_count))
self.logger.info("rulez 抓取时间{0}，数据遍历时间{1}".format(
crawl_time, storage_time))
def Add_Data_To_Url(urls):
for url in urls:
try:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
res_title = res.get('title')
res_power = res.get('power')
res_server = res.get('server')
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server)
except Exception as e:
print('错误代码 [29] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [29] {}'.format(str(e)))
try:
ip = get_host(url)
if ip == '获取失败':
return
print('URL --> {} IP --> {}'.format(url, ip))
URL.objects.create(url=url, ip=ip)
test_ip = list(IP.objects.filter(ip=ip))
if test_ip != []:
return
IP_Res = Get_Ip_Info(ip)
servers = IP_Res.get_server_from_nmap(ip)
# 服务与端口 字典类型
open_port = servers.keys()
check_alive_url = []
for port in open_port:
check_alive_url.append('http://{}:{}'.format(ip, port))
check_alive_url.append('https://{}:{}'.format(ip, port))
alive_url = Get_Alive_Url(check_alive_url)
# 该IP上存活WEB，类型为列表，内容为多个字典
host_type = IP_Res.get_host_type(ip)
# windows/linux
area = IP_Res.get_ip_address(ip)
# 返回地址
IP_Obj = IP()
IP_Obj.ip = ip
IP_Obj.servers = str(servers)
IP_Obj.host_type = host_type
IP_Obj.alive_urls = str(alive_url)
IP_Obj.area = area
try:
print(ip, servers, host_type, area)
except Exception as e:
print('错误代码 [34] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [34] {}'.format(str(e)))
IP_Obj.save()
except Exception as e:
print('错误代码 [30] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [30] {}'.format(str(e)))
# This is so my local_settings.py gets loaded.
os.chdir(proj_path)
# This is so models get loaded.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from app.models import IP
txtreader = open("time.txt",'r')
readtmp = "app/result/" + txtreader "ping.txt"
dataReader = csv.reader(open(readtmp), delimiter=',')
for row in dataReader:
if row[0] != 'Date/Time': # Ignore the header row, import everything else
ping = IP()
ping.date = row[0]
ping.site = row[1]
ping.tunName = row[2]
ping.tunType = row[3]
ping.ip = row[4]
ping.reach = row[5]
dtexists = IP.objects.filter(date__iexact = ping.date).exists()
diexists = IP.objects.filter(ip__iexact = ping.ip).exists()
if not diexists:
ping.save()
else:
pass