def test_row_retrieval(self):
ip_row = IP()
result = ip_row.find_by_ip(CURRENTIP)
result = result[0].data()
result.pop("time_stamp")
self.assertEqual(
result,
{
"request": "209.203.39.143",
"status": 200,
"delay": "1ms",
"credit": "Some of the returned data",
"city": "Bothasig",
"region": "Western Cape",
"region_code": "WC",
"region_name": "Western Cape",
"area_code": "",
"dma_code": "",
"country_code": "ZA",
"country_name": "South Africa",
"in_eu": "0",
"eu_vat_rate": "0",
"continent_code": "AF",
"continent_name": "Africa",
"latitude": "-33.8604",
"longitude": "18.5446",
"location_accuracy_radius": "500",
"timezone": "Africa/Johannesburg",
"currency_code": "ZAR",
"currency_symbol": "R",
"currency_converter": "14.1204",
},
)
self.conn.close()
def save_info(self, ips, source, crawl_time):
start = time.time()
all_count = len(ips)
avail_count = 0
if len(ips) > 0:
try:
for ip, updatetime, source in ips:
flag = db.session.query(IP).filter(
IP.ip == ip, IP.source == source).first()
if flag is None:
new_ip = IP()
new_ip.ip = ip
new_ip.updatetime = updatetime
new_ip.source = source
db.session.add(new_ip)
avail_count += 1
else:
flag.updatetime = updatetime
db.session.add(flag)
db.session.commit()
except Exception as e:
self.logger.warning("Error writing to database" + str(e) +
source)
else:
self.logger.warning("NO record found from: %s" % source)
stop = time.time()
storage_time = str(stop - start) + "秒"
self.logger.info("rulez 共收集{0}条数据， 新数据{1}条".format(
all_count, avail_count))
self.logger.info("rulez 抓取时间{0}，数据遍历时间{1}".format(
crawl_time, storage_time))
def find(ip_addr):
new_ip = IP()
row = new_ip.find_by_ip(ip_addr)
if not row:
logging.info("no entry found for %s", ip_addr)
new_ip = lookup(ip_addr)
logging.info(new_ip.data())
return
logging.info(row[0].data())
def Add_Data_To_Url(urls):
for url in urls:
try:
res = Get_Url_Info(url).get_info()
res_url = res.get('url')
res_title = res.get('title')
res_power = res.get('power')
res_server = res.get('server')
Other_Url.objects.create(url=res_url,
title=res_title,
power=res_power,
server=res_server)
except Exception as e:
print('错误代码 [29] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [29] {}'.format(str(e)))
try:
ip = get_host(url)
if ip == '获取失败':
return
print('URL --> {} IP --> {}'.format(url, ip))
URL.objects.create(url=url, ip=ip)
test_ip = list(IP.objects.filter(ip=ip))
if test_ip != []:
return
IP_Res = Get_Ip_Info(ip)
servers = IP_Res.get_server_from_nmap(ip)
# 服务与端口 字典类型
open_port = servers.keys()
check_alive_url = []
for port in open_port:
check_alive_url.append('http://{}:{}'.format(ip, port))
check_alive_url.append('https://{}:{}'.format(ip, port))
alive_url = Get_Alive_Url(check_alive_url)
# 该IP上存活WEB，类型为列表，内容为多个字典
host_type = IP_Res.get_host_type(ip)
# windows/linux
area = IP_Res.get_ip_address(ip)
# 返回地址
IP_Obj = IP()
IP_Obj.ip = ip
IP_Obj.servers = str(servers)
IP_Obj.host_type = host_type
IP_Obj.alive_urls = str(alive_url)
IP_Obj.area = area
try:
print(ip, servers, host_type, area)
except Exception as e:
print('错误代码 [34] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [34] {}'.format(str(e)))
IP_Obj.save()
except Exception as e:
print('错误代码 [30] {}'.format(str(e)))
Error_Log.objects.create(url=url,
error='错误代码 [30] {}'.format(str(e)))
def lookup(ip_addr):
url = f"http://www.geoplugin.net/json.gp?ip={ip_addr}"
try:
response = requests.get(url)
if response.status_code != 200:
logging.error("%s replyed with a not ok status code", url)
return ""
new_ip = IP()
new_ip.insert_raw(response.json())
new_ip.save()
return new_ip
except ConnectionError:
logging.error("failed to fetch api details for %s", url)
return ""
def test_insert(self):
new_ip = IP()
new_ip.insert_raw(APIRESPONSE)
new_ip.save()
# This is so my local_settings.py gets loaded.
os.chdir(proj_path)
# This is so models get loaded.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from app.models import IP
txtreader = open("time.txt",'r')
readtmp = "app/result/" + txtreader "ping.txt"
dataReader = csv.reader(open(readtmp), delimiter=',')
for row in dataReader:
if row[0] != 'Date/Time': # Ignore the header row, import everything else
ping = IP()
ping.date = row[0]
ping.site = row[1]
ping.tunName = row[2]
ping.tunType = row[3]
ping.ip = row[4]
ping.reach = row[5]
dtexists = IP.objects.filter(date__iexact = ping.date).exists()
diexists = IP.objects.filter(ip__iexact = ping.ip).exists()
if not diexists:
ping.save()
else:
pass
def login(request, template_name='app/login.html',
redirect_after=None,
demo_account=None,
redirect_field_name=REDIRECT_FIELD_NAME,
api_called=False):
if request.user.is_authenticated():
return HttpResponseRedirect(reverse('school_index'))
"""
login the system
"""
redirect_to = request.REQUEST.get(redirect_field_name, '')
error = None
error_type = None
login_failure_exceed = False
if request.method == "POST":
if api_called:
form = AuthenticationForm(data=request.POST,
request=request,
api_called=True)
else:
if demo_account: request.POST['login_type'] = demo_account
form = AuthenticationForm(data=request.POST, request=request)
if form.is_valid():
netloc = urlparse.urlparse(redirect_to)[1]
# Use default setting if redirect_to is empty
if not redirect_to:
if not redirect_after:
redirect_to = settings.LOGIN_REDIRECT_URL
else: redirect_to = urllib.unquote(urllib.unquote(redirect_after))
# Security check -- don't allow redirection to a different
# host.
elif netloc and netloc != request.get_host():
redirect_to = settings.LOGIN_REDIRECT_URL
# Okay, security checks complete. Log the user in.
auth_login(request, form.get_user())
#TODO: notice IP and tell user if the IP is stranger
if not request.user.is_anonymous():
IP.reset(request.META['REMOTE_ADDR'])
request.session.set_expiry(0)
log_action(request, form.get_user(), "logged in")
elif api_called: return HttpResponse(status=401)
if request.session.test_cookie_worked():
request.session.delete_test_cookie()
if api_called: return HttpResponse(status=200)
return HttpResponseRedirect(redirect_to)
else:
error = form._errors
error_type = form.errors_type
try:
IP.recognize(request.META['REMOTE_ADDR'], request.user)
except Exception as e:
if e.message == 'LoginFailureExceed': login_failure_exceed = True
if api_called: return HttpResponse(status=401)
else:
if IP.exceed_failure_limit(request.META['REMOTE_ADDR']):
login_failure_exceed = True
form = AuthenticationForm(request)
if api_called:
request.session.set_test_cookie()
return HttpResponse(content=simplejson.dumps({
'csrf_token':get_token(request)}), mimetype='json')
request.session.set_test_cookie()
context = {
'form': form,
'error': error,
'error_type': error_type,
'login_failure_exceed': login_failure_exceed,
redirect_field_name: redirect_to
}
return render_to_response(template_name, context,
context_instance=django.template.RequestContext(request))
def list_entries():
new_ip = IP()
rows = new_ip.list_all()
print(rows)
for row in rows:
logging.info(row.data())