class IPLocationHandler(BaseHandler):
"""处理 IP 消息，返回 IP 地理位置
"""
command = 'ip'
def __init__(self, wx_client=None):
"""获取 IP 地址数据
"""
super().__init__()
file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../static/qqwry.dat')
self.q = QQwry()
self.q.load_file(file)
def handle(self, message):
# 判断消息开头是否匹配
if not self.check_match(message):
return
parts = message.content.strip().split()
if len(parts) == 1 or len(parts) > 2:
return create_reply('IP 地址无效', message)
ip = parts[1]
pattern = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
if not re.match(pattern, ip):
return create_reply('IP 地址无效', message)
result = self.q.lookup(ip)
if result:
return create_reply(result[0], message)
return create_reply('未找到 IP 对应的地址', message)
def handle(self, message):
if not self.check_match(message):
return
parts = message.content.strip().split()
if len(parts) == 1 or len(parts) > 2:
return create_reply('command wrong, correct format ip 123.123.123.123', message)
ip = parts[1]
print (ip)
patterns = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
if not re.match(patterns, ip):
return create_reply('wrong ip addresss',message)
file = '/Users/yangedwin/workspace/week6/rmon/qqwry.dat'
q = QQwry()
q.load_file(file)
result = q.lookup(ip)
print (result)
if result is None:
return create_reply('could not find this address',message)
else:
return create_reply(result[0],message)
def load_qqwry():
ipq = QQwry()
res = ipq.load_file(IP_DATA_FILE, loadindex=True)
if not res:
print("qqwry load failed", IP_DATA_FILE)
sys.exit(-1)
return ipq
def __init__(self, wx_client=None):
"""获取 IP 地址数据
"""
super().__init__()
file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../static/qqwry.dat')
self.q = QQwry()
self.q.load_file(file)
class Fetcher(object):
name = "fetcher"
def __init__(self):
self.log = LogHandler(self.name)
self.conf = ConfigHandler()
self.proxy_handler = ProxyHandler()
self.loadIp()
def loadIp(self):
if False != os.path.isfile("qqwry.dat"):
self.ip = QQwry()
self.ip.load_file('qqwry.dat')
else:
self.ip = False
def fetch(self):
"""
fetch proxy into db with proxyFetcher
:return:
"""
proxy_set = set()
self.log.info("ProxyFetch : start")
for fetch_name in self.conf.fetchers:
self.log.info("ProxyFetch - {func}: start".format(func=fetch_name))
fetcher = getattr(ProxyFetcher, fetch_name, None)
if not fetcher:
self.log.error("ProxyFetch - {func}: class method not exists!")
continue
if not callable(fetcher):
self.log.error("ProxyFetch - {func}: must be class method")
continue
try:
for proxy in fetcher():
if proxy in proxy_set:
self.log.info('ProxyFetch - %s: %s exist' %
(fetch_name, proxy.ljust(23)))
continue
else:
self.log.info('ProxyFetch - %s: %s success' %
(fetch_name, proxy.ljust(23)))
if proxy.strip():
if self.ip:
area = " ".join(self.ip.lookup(
proxy.split(':')[0]))
else:
self.loadIp()
area = ''
proxy_set.add((proxy, fetch_name, area))
except Exception as e:
self.log.error(
"ProxyFetch - {func}: error".format(func=fetch_name))
self.log.error(str(e))
self.log.info("ProxyFetch - all complete!")
return proxy_set
def get_country(proxy):
q = QQwry()
q.load_file('../qqwry.dat')
list = proxy.split(':')
ip = list[1].strip('/')
country = q.lookup(ip)
if (country):
return country[0]
return None
def add_scan_record(qr_label, ip):
q = QQwry()
sr = ScanRecord()
sr.ip = ip
sr.qr_label = qr_label
if q.load_file(settings.QQPATH):
sr.json = q.lookup(str(ip))
sr.city = sr.json[0]
sr.save()
def getip_info_local(ip):
try:
q = QQwry()
q.load_file('./qqwry.dat')
result = q.lookup(ip)
area = result[0]
isp = result[1]
return area, isp, True
except Exception:
return '', '', False
class IP2Region():
'''
解析ip地址：lookup(ip)方法将ip转化成（"省份","地区"）
'''
def __init__(self):
self.filename = 'qqwry.dat'
self.dat_path = os.path.join(base_config.BASE_PATH, 'utils', self.filename)
self.q = QQwry()
if os.path.exists(self.dat_path):
self.q.load_file(self.dat_path)
print(self.q)
else:
print('初始化更新ip库')
self.update_dat()
self.reload_dat()
return
def get_lastone(self):
'''
返回最后一条数据，最后一条通常为数据的版本号
没有数据则返回一个None
:return:
'''
version = self.q.get_lastone()
return version
def update_dat(self):
'''
异步更新，使用线程或者celery更新IP数据库源
:return:(bool) 成功后返回一个正整数，是文件的字节数；失败则返回一个负整数。
'''
result = updateQQwry(self.dat_path)
if result > 0:
print('ip库更新成功')
return True
print('ip库更新失败，网络出现故障')
return False
def lookup(self, ip):
'''
解析ip地址
:param ip(str): 要解析的ip地址
:return: 找到则返回一个含有两个字符串的元组，如：('国家', '省份')没有找到结果，则返回一个None
'''
return self.q.lookup(ip)
def reload_dat(self):
'''
重载IP数据源，当IPdat数据更新的时候，使用此方法更新数据
:return: (bool) 重载成功返回True，重载失败返回Flase
'''
self.q.clear()
if not self.q.is_loaded():
self.q.load_file(self.dat_path)
return True
return False
def __init__(self):
self.filename = 'qqwry.dat'
self.dat_path = os.path.join(base_config.BASE_PATH, 'utils', self.filename)
self.q = QQwry()
if os.path.exists(self.dat_path):
self.q.load_file(self.dat_path)
print(self.q)
else:
print('初始化更新ip库')
self.update_dat()
self.reload_dat()
return
def handle_ip(ip):
q = QQwry()
q.load_file(QQwrydat.path)
_ = q.lookup(ip)
with PeiZhi() as db:
data = db.get_peizhi()
if data['need_switch'] == 1:
now = time.strftime("%H:%M", time.localtime())
if now >= data['switch_time_start'] and now <= data['switch_time_end']:
with Sheng() as db:
for i in db.get_need_switch():
if i in _[0]:
return True
return False
def handle(self, message):
if not self.check_match(message):
return
lookup_data = message.content.strip().split()
if len(lookup_data) == 1 or len(lookup_data) > 2:
return create_reply("invalid IP", message)
ip = lookup_data[1]
if not re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',ip):
return create_reply("IP invalid", message)
q = QQwry()
q.load_file('qqwry.dat')
reply = q.lookup(ip)
if reply is None:
return None
return create_reply(reply[0], message)
class qqwry_search(object):
"""docstring for qqwry_search"""
def __new__(cls, *args, **kwargs):
if not hasattr(cls, "instance"):
cls.instance = super(qqwry_search, cls).__new__(cls)
return cls.instance
def __init__(self):
from qqwry import QQwry
self.q = QQwry()
self.q.load_file('qqwry.dat')
def __call__(self, ip: str):
return ' '.join(self.q.lookup(ip))
def __del__(self):
self.q.clear()
def listSql():
page = request.args.get('page', 1, type=int)
listt = Role.query.all()
pagination = IpList.query.order_by(IpList.time.desc()).paginate(
page, per_page=15, error_out=False)
ip = pagination.items
for i in ip:
ip_data = i.ip
q = QQwry()
filename = os.path.join(STATIC_DIR, 'qqwry.dat')
q.load_file(filename, loadindex=False)
adders = q.lookup(ip_data)
query_ip = IpList.query.order_by(
IpList.time.desc()).filter_by(ip=ip_data).first()
query_ip.adders = str(adders)
db.session.commit()
return render_template('list.html',
ip=ip,
pagination=pagination,
listt=listt)
def get_city_of_ip(ip):
"""根据IP地址获取城市名称"""
q = QQwry()
res = q.load_file(config.IP_DATABASE_FILENAME, loadindex=False)
if res:
result = q.lookup(ip)
q.clear()
return result[0]
class IPLocationHandler(CommandHandler):
command = 'ip'
def __init__(self):
file = os.environ.get('QQWRY_DAT', 'qqwry.dat')
self.q = QQwry()
self.q.load_file(file)
def handle(self, message):
if not self.check_match(message):
return
parts = message.content.strip().split()
if len(parts) == 1 or len(parts) > 2:
return create_reply('IP地址无效', message)
ip = parts[1]
pattern = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
if not re.match(pattern, ip):
return create_reply('IP地址无效', message)
result = self.q.lookup(ip)
if result is None:
return create_reply('未找到', message)
else:
return create_reply(result[0], message)
class IPLocationHandler(CommandHandler):
command = 'ip'
def __init__(self):
file = os.environ.get('QQWRY_DAT', 'qqwry.dat')
self.q = QQwry()
self.q.load_file(file)
def handle(self, message):
if not self.check_match(message):
return
parts = message.content.strip().split()
if len(parts) == 1 or len(parts) > 2:
return create_reply('IP地址无效', message)
ip = parts[1]
pattern = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
if not re.match(pattern, ip):
return create_reply('IP地址无效', message)
result = self.q.lookup(ip)
if result is None:
return create_reply('未找到', message)
else:
return create_reply(result[0], message)
def get_raw_data():
"""
得到raw_data
:return:返回预处理后的数据
"""
filename = LOG_PATH
qqwry_path = QQWRY_PATH
ip_dict = {} # 存不重复的ip及其city
log_text = get_log_text(filename) # 获取日志原始文件
parsed_log_list = parse(log_text) # 获取解析完的日志
# 加载纯真ip数据库
flag = 0
q = QQwry()
while not q.is_loaded():
# 加载3次失败后改用其他方案
if flag == 3:
break
q.load_file(qqwry_path)
flag += 1
raw_data = []
Ip_raw_data = namedtuple(
'ip_data',
['log_time', 'ip', 'referrer', 'client', 'method', 'status', 'city'])
for line in parsed_log_list:
log_time = parse_nginx_date(line['date'] + ':' + line['time'],
line['tz'])
log_ip = line['origin']
if log_ip not in ip_dict:
ip_dict[log_ip] = get_address_by_ip(line, q, flag)
log_city = ip_dict[log_ip]
raw_data.append(
Ip_raw_data(log_time, log_ip, line['referrer'], line['client'],
line['method'], line['status'], log_city))
return raw_data
def batch_query_and_print():
county = set()
vps = set()
ip = set()
q = QQwry()
q.load_file('qqwry.dat')
with open('ip1.txt') as f:
ip_list = f.read().splitlines()
for read_content in ip_list:
try:
IP(read_content)
except:
print("有不符合规范的IP地址，请检查后重新运行")
exit(0)
address_list = [q.lookup(ip) for ip in ip_list]
for i, j in zip(ip_list, address_list):
query_results = i + " " + j[0] + " " + j[1]
county.add(j[0])
vps.add(j[1])
ip.add(i)
print(len(ip), len(county), len(vps))
with open("query_results.txt", 'a', encoding='utf-8') as f:
f.writelines(query_results + "\n")
print(query_results)
# coding=utf8
from qqwry import QQwry
from datetime import datetime
from ua_parser import user_agent_parser
from settings import HOST
from urllib.parse import quote
from settings import MONGODB_HOST, MONGODB_PORT, DATABASE, COLLECTION
from pymongo import MongoClient
q = QQwry()
q.load_file('qqwry.dat', loadindex=False)
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = client[DATABASE]
collection = db[COLLECTION]
def analyse_client(ip, ua):
data = {}
data['time'] = datetime.now()
data['location'] = get_loc_from_ip(ip)
data['client'] = user_agent_parser.Parse(ua)
collection.insert(data)
def get_loc_from_ip(ip):
loc, supplier = q.lookup(ip)
return {
'ip': ip,
'loc': loc,
'''
Author: Dong Xing
Date: 2021-01-15 20:05:06
LastEditors: Dong Xing
LastEditTime: 2021-01-17 02:58:51
Description: file content
'''
from fastapi import Query, APIRouter, Request
from qqwry import QQwry
q = QQwry()
q.load_file('./data/qqwry.dat')
router = APIRouter()
@router.get("/ipcity/")
async def get_ipcity(ip: str):
# 数据来源：https://github.com/ipipdotnet/ipdb-php
result = q.lookup(ip)
if result:
return {'code': 200, 'city': result[0], 'operator': result[1]}
else:
return {'code': 404, 'city': None, 'operator': None}
@router.get("/ip/")
async def read_root(request: Request):
client_host = request.client.host
return {"client_host": client_host}
import xlrd
from qqwry import QQwry
q = QQwry()
# 加载纯真镜像 dat 文件
# https://github.com/WisdomFusion/qqwry.dat
q.load_file('F:\qqwry.dat')
# 打开存储 IP 的 excel 表格
wb = xlrd.open_workbook(filename="IP.xlsx")
# 对应到目标表
ip = wb.sheet_by_index(0)
# 从上到下，按列
rows = ip.nrows
# 将结果保存在 result.txt 中
ipResult = open("result.txt", 'w')
for i in range(rows):
# 得到每一个 IP
ip_value = ip.cell(i, 0).value
# 查询
result = q.lookup(ip_value)
print(result)
city = result[0]
isp = result[1]
ipResult.write(ip_value + '\t' + city + '\t' + isp + '\r\n')
ipResult.close()
class SQLite3Client:
"""
代理池依赖了 Redis 数据库，使用了其`有序集合`的数据结构
（可按分数排序，key 值不能重复）
"""
def __init__(self, FILENAME=SQLITE3_FILENAME):
self.conn = g_conn #sqlite3.connect(FILENAME)
self.cursor = self.conn.cursor()
self.wry = QQwry()
self.wry.load_file(QQWRY_FILENAME)
def add_proxy(self, proxy, score=INIT_SCORE):
"""
新增一个代理，初始化分数 INIT_SCORE < MAX_SCORE，确保在
运行完收集器后还没运行校验器就获取代理，导致获取到分数虽为 MAX_SCORE,
但实际上确是未经验证，不可用的代理
:param proxy: 新增代理
:param score: 初始化分数
"""
country = self.get_country(proxy)
type = self.get_proxy_type(proxy)
sql = "INSERT INTO Proxy (PROXY, TYPE, COUNTRY, SPEED, SCORE) VALUES('%s', '%s', '%s', %f, %d)"\
% (proxy, type, country, 100.0, score)
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception:
pass
def reduce_proxy_score(self, proxy):
"""
验证未通过，分数减一
:param proxy: 验证代理
"""
sql = "select SCORE from Proxy where PROXY=='%s'" % proxy
cur = self.cursor.execute(sql)
value = cur.fetchone()
score = int(value[0])
if score and score > MIN_SCORE:
score -= 1
sql = "UPDATE Proxy set SCORE=%d where PROXY=='%s'" %(score, proxy)
else:
sql = "DELETE from Proxy where PROXY=='%s'" % proxy
self.cursor.execute(sql)
self.conn.commit()
def increase_proxy_score(self, proxy):
"""
验证通过，分数加一
:param proxy: 验证代理
"""
sql = "select SCORE from Proxy where PROXY=='%s'" % proxy
cur = self.cursor.execute(sql)
value = cur.fetchone()
score = int(value[0])
if score and score < MAX_SCORE:
score += 1
sql = "UPDATE Proxy set SCORE=%d where PROXY=='%s'" % (score, proxy)
self.cursor.execute(sql)
self.conn.commit()
def update_proxy_speed(self, proxy, speed):
try:
sql = "UPDATE Proxy set SPEED=%f, TIME = datetime('now','localtime') where PROXY=='%s'" % (speed, proxy)
self.cursor.execute(sql)
self.conn.commit()
return True
except Exception:
return False
def pop_proxy(self):
"""
返回一个代理
"""
# 第一次尝试取分数最高，也就是最新可用的代理
sql = "SELECT PROXY FROM Proxy ORDER BY SCORE DESC LIMIT 1"
cur = self.cursor.execute(sql)
value = cur.fetchone()
if value:
return value[0]
return 0
def get_proxies(self, count=1):
"""
返回指定数量代理，分数由高到低排序
:param count: 代理数量
"""
#print("count:%d" %count)
sql = "SELECT PROXY FROM Proxy ORDER BY SCORE DESC LIMIT %d" % count
cur = self.cursor.execute(sql)
#print(cur.fetchall())
for row in cur.fetchall():
yield row[0]
def count_all_proxies(self):
"""
返回所有代理总数
"""
sql = "SELECT COUNT(*) FROM Proxy"
cur = self.cursor.execute(sql)
value = cur.fetchone()
if value:
return value[0]
return 0
def count_score_proxies(self, score):
"""
返回指定分数代理总数
:param score: 代理分数
"""
if 0 <= score <= 10:
sql = "SELECT COUNT(*) FROM Proxy WHERE SCORE==%d" % score
cur = self.cursor.execute(sql)
value = cur.fetchone()
if value:
return value[0]
return -1
def get_score_proxies(self, score):
"""
返回数据库的中制定的score的所有数据
:param score: 代理分数
"""
list = []
sql = "SELECT PROXY FROM Proxy WHERE SCORE== %d" % score
values = self.cursor.execute(sql)
for row in values:
list.append(row[0])
return list
def clear_proxies(self, score):
"""
删除分数小于等于 score 的代理
"""
if 0 <= score <= 10:
sql = "DELETE from Proxy WHERE SCORE <= %d" % score
self.cursor.execute(sql)
self.conn.commit()
return True
return False
def get_country(self, proxy):
pro_split_list = proxy.split(':')
ip = pro_split_list[1].strip('/')
ipcountry = self.wry.lookup(ip)
if ipcountry:
return ipcountry[0]
return None
def get_proxy_type(self, proxy):
pro_split_list = proxy.split(':')
if pro_split_list[0].find('http') >=0:
return pro_split_list[0]
return None
def all_proxies(self):
"""
返回全部代理
"""
list=[]
sql = "SELECT PROXY FROM Proxy"
values = self.cursor.execute(sql)
for row in values:
list.append(row[0])
return list
def __del__(self):
self.conn.close()
# if __name__=='__main__':
# #test_main(proxyAddressArray)
# gou = SQLite3Client()
# proxy_all_list = gou.get_proxies(1)
# for proxy in proxy_all_list:
# print(proxy)
# print("pop: %s" % gou.pop_proxy())
# #gou.clear_proxies(9)
# print("count_score_proxies:%d" %gou.count_score_proxies(9))
# print("count_all_proxies:%d" % gou.count_all_proxies())
# proxy_val = 'http://103.199.159.177:40049'
# #gou.increase_proxy_score(proxy_val)
# gou.reduce_proxy_score(proxy_val)
# #gou.add_proxy()
def __init__(self, FILENAME=SQLITE3_FILENAME):
self.conn = g_conn #sqlite3.connect(FILENAME)
self.cursor = self.conn.cursor()
self.wry = QQwry()
self.wry.load_file(QQWRY_FILENAME)
from qqwry import QQwry
q = QQwry()
#filename可以是qqwry.dat的文件名也可以是bytes类型的文件内容
#当参数loadindex=False时 加载速度快进程内存少查询速度慢
#当参数loadindex=True时 加载速度慢进程内存多查询速度快
q.load_file('qqwry.dat', loadindex=False)
result = q.lookup('8.8.8.8')
print(result)
#!/usr/bin/python
import base64
import time
import sys, os
import socket
from qqwry import QQwry
q = QQwry()
q.load_file('qqwry/qqwry.dat')
configfilepath = 'configfile/'
def ToBase64(file, txt):
with open(file, 'rb') as fileObj:
image_data = fileObj.read()
base64_data = base64.b64encode(image_data)
fout = open(txt, 'w')
fout.write(base64_data.decode())
fout.close()
def ToFile(txt, file):
with open(txt, 'r') as fileObj:
base64_data = fileObj.read()
ori_image_data = base64.b64decode(base64_data)
fout = open(file, 'wb')
fout.write(ori_image_data)
fout.close()
#ToBase64("./desk.jpg",'desk_base64.txt') # 文件转换为base64
def __init__(self):
from qqwry import QQwry
self.q = QQwry()
self.q.load_file('qqwry.dat')
def main(i):
L = []
path = r'F:\ip\{0}_ip.csv'.format(i)
for ind,dt in enumerate(gen_data(path)):
if len(L) > 200000: #性能最优解,一分钟可插入30万条数据
print('Saving {0} rows data...'.format(len(L)))
save_data(L)
L = []
else:
L.append(dt)
print(ind, dt)
if __name__ == '__main__':
q = QQwry()
q.load_file('qqwry.dat')
conn = pymysql.connect(host='xxx.xx.xx.xx',
port=33066,
user='******',
password='******',
database='dbxxx',
charset='utf8mb4')
#cursorclass=pymysql.cursors.DictCursor)
cur = conn.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS dbxxx.addr920 (userid int PRIMARY KEY,address text,network text);")
try:
main(2014)#TODO
finally:
cur.close()
########################################### 纯真ip ######################################
#https://github.com/animalize/qqwry-python3
#https://github.com/gwind/ylinux/tree/master/tools/IP/QQWry
#https://segmentfault.com/a/1190000000352578
##更新本地dat文件
#from qqwry import updateQQwry
#result = updateQQwry('qqwry.dat')
##use
from qqwry import QQwry
q = QQwry()
q.load_file('qqwry.dat')#工作路径
result = q.lookup('xxx.xxx.xx.xx')
print(result)
###################################### ipip.net #########################################
#https://github.com/17mon/python
import os
from ipip import IP#目前无法pip(python3.5.1)
from ipip import IPX
IP.load(os.path.abspath("mydata4vipday2.dat"))#工作路径
print IP.find("118.28.8.8")
IPX.load(os.path.abspath("mydata4vipday2.datx"))#工作路径
print IPX.find("118.28.8.8")
def loadIp(self):
if False != os.path.isfile("qqwry.dat"):
self.ip = QQwry()
self.ip.load_file('qqwry.dat')
else:
self.ip = False
@Blog: www.langzi.fun
@time: 2019/8/5 22:47
@file: 获取IP信息.py
"""
import IPy
import nmap
import socket
socket.setdefaulttimeout(25)
from urllib.parse import urlparse
import requests
requests.packages.urllib3.disable_warnings()
import time
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from qqwry import QQwry
from scapy.all import *
q = QQwry()
import os
q.load_file(os.path.join('Auxiliary', 'IP_ADDRESS.dat'))
import django
import os
import sys
pathname = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, pathname)
sys.path.insert(0, os.path.abspath(os.path.join(pathname, '..')))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "LangSrcCurise.settings")
django.setup()
from app.models import Setting, Error_Log
Set = Setting.objects.all()[0]
pool_count = int(Set.Pool)
def __init__(self):
file = os.environ.get('QQWRY_DAT', 'qqwry.dat')
self.q = QQwry()
self.q.load_file(file)
