class IPLocationHandler(BaseHandler): """处理 IP 消息,返回 IP 地理位置 """ command = 'ip' def __init__(self, wx_client=None): """获取 IP 地址数据 """ super().__init__() file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../static/qqwry.dat') self.q = QQwry() self.q.load_file(file) def handle(self, message): # 判断消息开头是否匹配 if not self.check_match(message): return parts = message.content.strip().split() if len(parts) == 1 or len(parts) > 2: return create_reply('IP 地址无效', message) ip = parts[1] pattern = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$' if not re.match(pattern, ip): return create_reply('IP 地址无效', message) result = self.q.lookup(ip) if result: return create_reply(result[0], message) return create_reply('未找到 IP 对应的地址', message)
def handle(self, message): if not self.check_match(message): return parts = message.content.strip().split() if len(parts) == 1 or len(parts) > 2: return create_reply('command wrong, correct format ip 123.123.123.123', message) ip = parts[1] print (ip) patterns = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$' if not re.match(patterns, ip): return create_reply('wrong ip addresss',message) file = '/Users/yangedwin/workspace/week6/rmon/qqwry.dat' q = QQwry() q.load_file(file) result = q.lookup(ip) print (result) if result is None: return create_reply('could not find this address',message) else: return create_reply(result[0],message)
def load_qqwry(): ipq = QQwry() res = ipq.load_file(IP_DATA_FILE, loadindex=True) if not res: print("qqwry load failed", IP_DATA_FILE) sys.exit(-1) return ipq
def __init__(self, wx_client=None): """获取 IP 地址数据 """ super().__init__() file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../static/qqwry.dat') self.q = QQwry() self.q.load_file(file)
class Fetcher(object): name = "fetcher" def __init__(self): self.log = LogHandler(self.name) self.conf = ConfigHandler() self.proxy_handler = ProxyHandler() self.loadIp() def loadIp(self): if False != os.path.isfile("qqwry.dat"): self.ip = QQwry() self.ip.load_file('qqwry.dat') else: self.ip = False def fetch(self): """ fetch proxy into db with proxyFetcher :return: """ proxy_set = set() self.log.info("ProxyFetch : start") for fetch_name in self.conf.fetchers: self.log.info("ProxyFetch - {func}: start".format(func=fetch_name)) fetcher = getattr(ProxyFetcher, fetch_name, None) if not fetcher: self.log.error("ProxyFetch - {func}: class method not exists!") continue if not callable(fetcher): self.log.error("ProxyFetch - {func}: must be class method") continue try: for proxy in fetcher(): if proxy in proxy_set: self.log.info('ProxyFetch - %s: %s exist' % (fetch_name, proxy.ljust(23))) continue else: self.log.info('ProxyFetch - %s: %s success' % (fetch_name, proxy.ljust(23))) if proxy.strip(): if self.ip: area = " ".join(self.ip.lookup( proxy.split(':')[0])) else: self.loadIp() area = '' proxy_set.add((proxy, fetch_name, area)) except Exception as e: self.log.error( "ProxyFetch - {func}: error".format(func=fetch_name)) self.log.error(str(e)) self.log.info("ProxyFetch - all complete!") return proxy_set
def get_country(proxy): q = QQwry() q.load_file('../qqwry.dat') list = proxy.split(':') ip = list[1].strip('/') country = q.lookup(ip) if (country): return country[0] return None
def add_scan_record(qr_label, ip): q = QQwry() sr = ScanRecord() sr.ip = ip sr.qr_label = qr_label if q.load_file(settings.QQPATH): sr.json = q.lookup(str(ip)) sr.city = sr.json[0] sr.save()
def getip_info_local(ip): try: q = QQwry() q.load_file('./qqwry.dat') result = q.lookup(ip) area = result[0] isp = result[1] return area, isp, True except Exception: return '', '', False
class IP2Region(): ''' 解析ip地址:lookup(ip)方法将ip转化成("省份","地区") ''' def __init__(self): self.filename = 'qqwry.dat' self.dat_path = os.path.join(base_config.BASE_PATH, 'utils', self.filename) self.q = QQwry() if os.path.exists(self.dat_path): self.q.load_file(self.dat_path) print(self.q) else: print('初始化更新ip库') self.update_dat() self.reload_dat() return def get_lastone(self): ''' 返回最后一条数据,最后一条通常为数据的版本号 没有数据则返回一个None :return: ''' version = self.q.get_lastone() return version def update_dat(self): ''' 异步更新,使用线程或者celery更新IP数据库源 :return:(bool) 成功后返回一个正整数,是文件的字节数;失败则返回一个负整数。 ''' result = updateQQwry(self.dat_path) if result > 0: print('ip库更新成功') return True print('ip库更新失败,网络出现故障') return False def lookup(self, ip): ''' 解析ip地址 :param ip(str): 要解析的ip地址 :return: 找到则返回一个含有两个字符串的元组,如:('国家', '省份')没有找到结果,则返回一个None ''' return self.q.lookup(ip) def reload_dat(self): ''' 重载IP数据源,当IPdat数据更新的时候,使用此方法更新数据 :return: (bool) 重载成功返回True,重载失败返回Flase ''' self.q.clear() if not self.q.is_loaded(): self.q.load_file(self.dat_path) return True return False
def __init__(self): self.filename = 'qqwry.dat' self.dat_path = os.path.join(base_config.BASE_PATH, 'utils', self.filename) self.q = QQwry() if os.path.exists(self.dat_path): self.q.load_file(self.dat_path) print(self.q) else: print('初始化更新ip库') self.update_dat() self.reload_dat() return
def handle_ip(ip): q = QQwry() q.load_file(QQwrydat.path) _ = q.lookup(ip) with PeiZhi() as db: data = db.get_peizhi() if data['need_switch'] == 1: now = time.strftime("%H:%M", time.localtime()) if now >= data['switch_time_start'] and now <= data['switch_time_end']: with Sheng() as db: for i in db.get_need_switch(): if i in _[0]: return True return False
def handle(self, message): if not self.check_match(message): return lookup_data = message.content.strip().split() if len(lookup_data) == 1 or len(lookup_data) > 2: return create_reply("invalid IP", message) ip = lookup_data[1] if not re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',ip): return create_reply("IP invalid", message) q = QQwry() q.load_file('qqwry.dat') reply = q.lookup(ip) if reply is None: return None return create_reply(reply[0], message)
class qqwry_search(object): """docstring for qqwry_search""" def __new__(cls, *args, **kwargs): if not hasattr(cls, "instance"): cls.instance = super(qqwry_search, cls).__new__(cls) return cls.instance def __init__(self): from qqwry import QQwry self.q = QQwry() self.q.load_file('qqwry.dat') def __call__(self, ip: str): return ' '.join(self.q.lookup(ip)) def __del__(self): self.q.clear()
def listSql(): page = request.args.get('page', 1, type=int) listt = Role.query.all() pagination = IpList.query.order_by(IpList.time.desc()).paginate( page, per_page=15, error_out=False) ip = pagination.items for i in ip: ip_data = i.ip q = QQwry() filename = os.path.join(STATIC_DIR, 'qqwry.dat') q.load_file(filename, loadindex=False) adders = q.lookup(ip_data) query_ip = IpList.query.order_by( IpList.time.desc()).filter_by(ip=ip_data).first() query_ip.adders = str(adders) db.session.commit() return render_template('list.html', ip=ip, pagination=pagination, listt=listt)
def get_city_of_ip(ip): """根据IP地址获取城市名称""" q = QQwry() res = q.load_file(config.IP_DATABASE_FILENAME, loadindex=False) if res: result = q.lookup(ip) q.clear() return result[0]
class IPLocationHandler(CommandHandler): command = 'ip' def __init__(self): file = os.environ.get('QQWRY_DAT', 'qqwry.dat') self.q = QQwry() self.q.load_file(file) def handle(self, message): if not self.check_match(message): return parts = message.content.strip().split() if len(parts) == 1 or len(parts) > 2: return create_reply('IP地址无效', message) ip = parts[1] pattern = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$' if not re.match(pattern, ip): return create_reply('IP地址无效', message) result = self.q.lookup(ip) if result is None: return create_reply('未找到', message) else: return create_reply(result[0], message)
def get_raw_data(): """ 得到raw_data :return:返回预处理后的数据 """ filename = LOG_PATH qqwry_path = QQWRY_PATH ip_dict = {} # 存不重复的ip及其city log_text = get_log_text(filename) # 获取日志原始文件 parsed_log_list = parse(log_text) # 获取解析完的日志 # 加载纯真ip数据库 flag = 0 q = QQwry() while not q.is_loaded(): # 加载3次失败后改用其他方案 if flag == 3: break q.load_file(qqwry_path) flag += 1 raw_data = [] Ip_raw_data = namedtuple( 'ip_data', ['log_time', 'ip', 'referrer', 'client', 'method', 'status', 'city']) for line in parsed_log_list: log_time = parse_nginx_date(line['date'] + ':' + line['time'], line['tz']) log_ip = line['origin'] if log_ip not in ip_dict: ip_dict[log_ip] = get_address_by_ip(line, q, flag) log_city = ip_dict[log_ip] raw_data.append( Ip_raw_data(log_time, log_ip, line['referrer'], line['client'], line['method'], line['status'], log_city)) return raw_data
def batch_query_and_print(): county = set() vps = set() ip = set() q = QQwry() q.load_file('qqwry.dat') with open('ip1.txt') as f: ip_list = f.read().splitlines() for read_content in ip_list: try: IP(read_content) except: print("有不符合规范的IP地址,请检查后重新运行") exit(0) address_list = [q.lookup(ip) for ip in ip_list] for i, j in zip(ip_list, address_list): query_results = i + " " + j[0] + " " + j[1] county.add(j[0]) vps.add(j[1]) ip.add(i) print(len(ip), len(county), len(vps)) with open("query_results.txt", 'a', encoding='utf-8') as f: f.writelines(query_results + "\n") print(query_results)
# coding=utf8 from qqwry import QQwry from datetime import datetime from ua_parser import user_agent_parser from settings import HOST from urllib.parse import quote from settings import MONGODB_HOST, MONGODB_PORT, DATABASE, COLLECTION from pymongo import MongoClient q = QQwry() q.load_file('qqwry.dat', loadindex=False) client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client[DATABASE] collection = db[COLLECTION] def analyse_client(ip, ua): data = {} data['time'] = datetime.now() data['location'] = get_loc_from_ip(ip) data['client'] = user_agent_parser.Parse(ua) collection.insert(data) def get_loc_from_ip(ip): loc, supplier = q.lookup(ip) return { 'ip': ip, 'loc': loc,
''' Author: Dong Xing Date: 2021-01-15 20:05:06 LastEditors: Dong Xing LastEditTime: 2021-01-17 02:58:51 Description: file content ''' from fastapi import Query, APIRouter, Request from qqwry import QQwry q = QQwry() q.load_file('./data/qqwry.dat') router = APIRouter() @router.get("/ipcity/") async def get_ipcity(ip: str): # 数据来源:https://github.com/ipipdotnet/ipdb-php result = q.lookup(ip) if result: return {'code': 200, 'city': result[0], 'operator': result[1]} else: return {'code': 404, 'city': None, 'operator': None} @router.get("/ip/") async def read_root(request: Request): client_host = request.client.host return {"client_host": client_host}
import xlrd from qqwry import QQwry q = QQwry() # 加载纯真镜像 dat 文件 # https://github.com/WisdomFusion/qqwry.dat q.load_file('F:\qqwry.dat') # 打开存储 IP 的 excel 表格 wb = xlrd.open_workbook(filename="IP.xlsx") # 对应到目标表 ip = wb.sheet_by_index(0) # 从上到下,按列 rows = ip.nrows # 将结果保存在 result.txt 中 ipResult = open("result.txt", 'w') for i in range(rows): # 得到每一个 IP ip_value = ip.cell(i, 0).value # 查询 result = q.lookup(ip_value) print(result) city = result[0] isp = result[1] ipResult.write(ip_value + '\t' + city + '\t' + isp + '\r\n') ipResult.close()
class SQLite3Client: """ 代理池依赖了 Redis 数据库,使用了其`有序集合`的数据结构 (可按分数排序,key 值不能重复) """ def __init__(self, FILENAME=SQLITE3_FILENAME): self.conn = g_conn #sqlite3.connect(FILENAME) self.cursor = self.conn.cursor() self.wry = QQwry() self.wry.load_file(QQWRY_FILENAME) def add_proxy(self, proxy, score=INIT_SCORE): """ 新增一个代理,初始化分数 INIT_SCORE < MAX_SCORE,确保在 运行完收集器后还没运行校验器就获取代理,导致获取到分数虽为 MAX_SCORE, 但实际上确是未经验证,不可用的代理 :param proxy: 新增代理 :param score: 初始化分数 """ country = self.get_country(proxy) type = self.get_proxy_type(proxy) sql = "INSERT INTO Proxy (PROXY, TYPE, COUNTRY, SPEED, SCORE) VALUES('%s', '%s', '%s', %f, %d)"\ % (proxy, type, country, 100.0, score) try: self.cursor.execute(sql) self.conn.commit() except Exception: pass def reduce_proxy_score(self, proxy): """ 验证未通过,分数减一 :param proxy: 验证代理 """ sql = "select SCORE from Proxy where PROXY=='%s'" % proxy cur = self.cursor.execute(sql) value = cur.fetchone() score = int(value[0]) if score and score > MIN_SCORE: score -= 1 sql = "UPDATE Proxy set SCORE=%d where PROXY=='%s'" %(score, proxy) else: sql = "DELETE from Proxy where PROXY=='%s'" % proxy self.cursor.execute(sql) self.conn.commit() def increase_proxy_score(self, proxy): """ 验证通过,分数加一 :param proxy: 验证代理 """ sql = "select SCORE from Proxy where PROXY=='%s'" % proxy cur = self.cursor.execute(sql) value = cur.fetchone() score = int(value[0]) if score and score < MAX_SCORE: score += 1 sql = "UPDATE Proxy set SCORE=%d where PROXY=='%s'" % (score, proxy) self.cursor.execute(sql) self.conn.commit() def update_proxy_speed(self, proxy, speed): try: sql = "UPDATE Proxy set SPEED=%f, TIME = datetime('now','localtime') where PROXY=='%s'" % (speed, proxy) self.cursor.execute(sql) self.conn.commit() return True except Exception: return False def pop_proxy(self): """ 返回一个代理 """ # 第一次尝试取分数最高,也就是最新可用的代理 sql = "SELECT PROXY FROM Proxy ORDER BY SCORE DESC LIMIT 1" cur = self.cursor.execute(sql) value = cur.fetchone() if value: return value[0] return 0 def get_proxies(self, count=1): """ 返回指定数量代理,分数由高到低排序 :param count: 代理数量 """ #print("count:%d" %count) sql = "SELECT PROXY FROM Proxy ORDER BY SCORE DESC LIMIT %d" % count cur = self.cursor.execute(sql) #print(cur.fetchall()) for row in cur.fetchall(): yield row[0] def count_all_proxies(self): """ 返回所有代理总数 """ sql = "SELECT COUNT(*) FROM Proxy" cur = self.cursor.execute(sql) value = cur.fetchone() if value: return value[0] return 0 def count_score_proxies(self, score): """ 返回指定分数代理总数 :param score: 代理分数 """ if 0 <= score <= 10: sql = "SELECT COUNT(*) FROM Proxy WHERE SCORE==%d" % score cur = self.cursor.execute(sql) value = cur.fetchone() if value: return value[0] return -1 def get_score_proxies(self, score): """ 返回数据库的中制定的score的所有数据 :param score: 代理分数 """ list = [] sql = "SELECT PROXY FROM Proxy WHERE SCORE== %d" % score values = self.cursor.execute(sql) for row in values: list.append(row[0]) return list def clear_proxies(self, score): """ 删除分数小于等于 score 的代理 """ if 0 <= score <= 10: sql = "DELETE from Proxy WHERE SCORE <= %d" % score self.cursor.execute(sql) self.conn.commit() return True return False def get_country(self, proxy): pro_split_list = proxy.split(':') ip = pro_split_list[1].strip('/') ipcountry = self.wry.lookup(ip) if ipcountry: return ipcountry[0] return None def get_proxy_type(self, proxy): pro_split_list = proxy.split(':') if pro_split_list[0].find('http') >=0: return pro_split_list[0] return None def all_proxies(self): """ 返回全部代理 """ list=[] sql = "SELECT PROXY FROM Proxy" values = self.cursor.execute(sql) for row in values: list.append(row[0]) return list def __del__(self): self.conn.close() # if __name__=='__main__': # #test_main(proxyAddressArray) # gou = SQLite3Client() # proxy_all_list = gou.get_proxies(1) # for proxy in proxy_all_list: # print(proxy) # print("pop: %s" % gou.pop_proxy()) # #gou.clear_proxies(9) # print("count_score_proxies:%d" %gou.count_score_proxies(9)) # print("count_all_proxies:%d" % gou.count_all_proxies()) # proxy_val = 'http://103.199.159.177:40049' # #gou.increase_proxy_score(proxy_val) # gou.reduce_proxy_score(proxy_val) # #gou.add_proxy()
def __init__(self, FILENAME=SQLITE3_FILENAME): self.conn = g_conn #sqlite3.connect(FILENAME) self.cursor = self.conn.cursor() self.wry = QQwry() self.wry.load_file(QQWRY_FILENAME)
from qqwry import QQwry q = QQwry() #filename可以是qqwry.dat的文件名也可以是bytes类型的文件内容 #当参数loadindex=False时 加载速度快进程内存少查询速度慢 #当参数loadindex=True时 加载速度慢进程内存多查询速度快 q.load_file('qqwry.dat', loadindex=False) result = q.lookup('8.8.8.8') print(result)
#!/usr/bin/python import base64 import time import sys, os import socket from qqwry import QQwry q = QQwry() q.load_file('qqwry/qqwry.dat') configfilepath = 'configfile/' def ToBase64(file, txt): with open(file, 'rb') as fileObj: image_data = fileObj.read() base64_data = base64.b64encode(image_data) fout = open(txt, 'w') fout.write(base64_data.decode()) fout.close() def ToFile(txt, file): with open(txt, 'r') as fileObj: base64_data = fileObj.read() ori_image_data = base64.b64decode(base64_data) fout = open(file, 'wb') fout.write(ori_image_data) fout.close() #ToBase64("./desk.jpg",'desk_base64.txt') # 文件转换为base64
def __init__(self): from qqwry import QQwry self.q = QQwry() self.q.load_file('qqwry.dat')
def main(i): L = [] path = r'F:\ip\{0}_ip.csv'.format(i) for ind,dt in enumerate(gen_data(path)): if len(L) > 200000: #性能最优解,一分钟可插入30万条数据 print('Saving {0} rows data...'.format(len(L))) save_data(L) L = [] else: L.append(dt) print(ind, dt) if __name__ == '__main__': q = QQwry() q.load_file('qqwry.dat') conn = pymysql.connect(host='xxx.xx.xx.xx', port=33066, user='******', password='******', database='dbxxx', charset='utf8mb4') #cursorclass=pymysql.cursors.DictCursor) cur = conn.cursor() cur.execute("CREATE TABLE IF NOT EXISTS dbxxx.addr920 (userid int PRIMARY KEY,address text,network text);") try: main(2014)#TODO finally: cur.close()
########################################### 纯真ip ###################################### #https://github.com/animalize/qqwry-python3 #https://github.com/gwind/ylinux/tree/master/tools/IP/QQWry #https://segmentfault.com/a/1190000000352578 ##更新本地dat文件 #from qqwry import updateQQwry #result = updateQQwry('qqwry.dat') ##use from qqwry import QQwry q = QQwry() q.load_file('qqwry.dat')#工作路径 result = q.lookup('xxx.xxx.xx.xx') print(result) ###################################### ipip.net ######################################### #https://github.com/17mon/python import os from ipip import IP#目前无法pip(python3.5.1) from ipip import IPX IP.load(os.path.abspath("mydata4vipday2.dat"))#工作路径 print IP.find("118.28.8.8") IPX.load(os.path.abspath("mydata4vipday2.datx"))#工作路径 print IPX.find("118.28.8.8")
def loadIp(self): if False != os.path.isfile("qqwry.dat"): self.ip = QQwry() self.ip.load_file('qqwry.dat') else: self.ip = False
@Blog: www.langzi.fun @time: 2019/8/5 22:47 @file: 获取IP信息.py """ import IPy import nmap import socket socket.setdefaulttimeout(25) from urllib.parse import urlparse import requests requests.packages.urllib3.disable_warnings() import time from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from qqwry import QQwry from scapy.all import * q = QQwry() import os q.load_file(os.path.join('Auxiliary', 'IP_ADDRESS.dat')) import django import os import sys pathname = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, pathname) sys.path.insert(0, os.path.abspath(os.path.join(pathname, '..'))) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "LangSrcCurise.settings") django.setup() from app.models import Setting, Error_Log Set = Setting.objects.all()[0] pool_count = int(Set.Pool)
def __init__(self): file = os.environ.get('QQWRY_DAT', 'qqwry.dat') self.q = QQwry() self.q.load_file(file)