def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = utils.split_ip(line)
nbegin = utils.ip_string_to_num(begin)
nend = utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
self.logger.warn("load ip range:%s fail", line)
continue
except Exception as e:
self.logger.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print utils.ip_num_to_string(nbegin), utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = utils.split_ip(line)
nbegin = utils.ip_string_to_num(begin)
nend = utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
self.logger.warn("load ip range:%s fail", line)
continue
except Exception as e:
self.logger.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append([nbegin, nend])
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print utils.ip_num_to_string(nbegin), utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def generate_bin(self):
self.logger.info("generating binary ip pool file.")
rfd = open(self.source_txt_fn, "rt")
wfd = open(self.dest_bin_fn, "wb")
num = 0
for line in rfd.readlines():
ip = line
try:
ip_num = utils.ip_string_to_num(ip)
except Exception as e:
self.logger.warn("ip %s not valid in %s", ip, self.source_txt_fn)
continue
ip_bin = struct.pack("<I", ip_num)
wfd.write(ip_bin)
num += 1
rfd.close()
wfd.close()
self.logger.info("finished generate binary ip pool file, num:%d", num)
def generate_bin(self):
self.logger.info("generating binary ip pool file.")
rfd = open(self.source_txt_fn, "rt")
wfd = open(self.dest_bin_fn, "wb")
num = 0
for line in rfd.readlines():
ip = line
try:
ip_num = utils.ip_string_to_num(ip)
except Exception as e:
self.logger.warn("ip %s not valid in %s", ip, self.source_txt_fn)
continue
ip_bin = struct.pack("<I", ip_num)
wfd.write(ip_bin)
num += 1
rfd.close()
wfd.close()
self.logger.info("finished generate binary ip pool file, num:%d", num)
def generate_db(self):
keeprange = (
'0.0.0.0/8', # 本地网络
'10.0.0.0/8', # 私有网络
'100.64.0.0/10', # 地址共享（运营商 NAT）
'127.0.0.0/8', # 环回地址
'169.254.0.0/16', # 链路本地
'172.16.0.0/12', # 私有网络
'192.0.0.0/24', # 保留地址（IANA）
'192.0.2.0/24', # TEST-NET-1
'192.88.99.0/24', # 6to4 中继
'192.168.0.0/16', # 私有网络
'198.18.0.0/15', # 网络基准测试
'198.51.100.0/24', # TEST-NET-2
'203.0.113.0/24', # TEST-NET-3
# 连续地址直到 IP 结束，特殊处理
# '224.0.0.0/4', #组播地址（D类）
# '240.0.0.0/4', #保留地址（E类）
)
keeplist = []
for iprange in keeprange:
ip, mask = iprange.split('/')
keeplist.append((utils.ip_string_to_num(ip), 32 - int(mask)))
mask_dict = dict((str(2**i), i) for i in range(8, 25))
def int2bytes2(n, pack=struct.pack):
'''将整数转换为大端序字节'''
return pack('>H', n)
# return bytes(map(lambda b: (-1 >> b & 255), (8, 0)))
def int2bytes4(n, pack=struct.pack):
'''将整数转换为大端序字节'''
return pack('>I', n)
# return bytes(map(lambda b: (n >> b & 255), (24, 16, 8, 0)))
def bytes2int(s):
nchars = len(s)
# string to int or long. Type depends on nchars
x = sum(
ord(s[byte]) << 8 * (nchars - byte - 1)
for byte in range(nchars))
return x
# +---------+
# | 4 bytes | <- data length
# +---------------+
# | 224 * 4 bytes | <- first ip number index
# +---------------+
# | 2n * 4 bytes | <- cn ip ranges data
# +------------------------+
# | b'end' and update info | <- end verify
# +------------------------+
lastip_s = 0
lastip_e = 0
index = {}
index_n = 0
index_fip = -1
offset = 0
padding = b'\xff\xff'
update = ""
iplist = []
fdi = open(self.cn_ipv4_range, "r")
for line in fdi.readlines():
lp = line.split()
iplist.append((utils.ip_string_to_num(lp[0]), mask_dict[lp[1]]))
iplist.extend(keeplist)
# 排序，不然无法处理
iplist.sort(key=lambda x: x[0])
# 随便算一下
buffering = len(iplist) * 8 + 224 * 4 + 64 + 4
buffer = bytearray(buffering)
for ip, mask in iplist:
ip_s = ip >> mask << mask
ip_e = (ip >> mask) + 1 << mask
# 判断连续
if ip_s <= lastip_e:
# 判断覆盖
if ip_e > lastip_e:
lastip_e = ip_e
continue
# 排除初始值
if lastip_e:
# 一段范围分为包含和排除
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
# 一个索引分为开始和结束
fip = ord(lastip_s[0:1]) * 2
if fip != index_fip:
# 前一个索引结束，序数多 1
# 避免无法搜索从当前索引结尾地址到下个索引开始地址
index[index_fip + 1] = index_b = int2bytes2(index_n)
# 当前索引开始
index[fip] = index_b
index_fip = fip
index_n += 2
offset += 8
lastip_s = ip_s
lastip_e = ip_e
# 添加最后一段范围
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
fip = ord(lastip_s[0:1]) * 2
if fip != index_fip:
index[index_fip + 1] = index_b = int2bytes2(index_n)
index[fip] = index_b
index_n += 2
offset += 8
# 添加最后一个结束索引
index[fip + 1] = int2bytes2(index_n)
# 写入文件
fd = open(self.cn_ipdb, 'wb', buffering)
fd.write(int2bytes4(offset))
for i in range(224 * 2):
fd.write(index.get(i, padding))
fd.write(buffer[:offset])
fd.write(b'endCN IP from ')
fd.write(update.encode('ascii'))
count = int(index_n // 2)
fd.write(b', range count: %d' % count)
fd.close()
xlog.debug('include IP range number：%s' % count)
xlog.debug('save to file:%s' % self.cn_ipdb)
def generate_db(self):
keeprange = (
'0.0.0.0/8', # 本地网络
'10.0.0.0/8', # 私有网络
'100.64.0.0/10', # 地址共享（运营商 NAT）
'127.0.0.0/8', # 环回地址
'169.254.0.0/16', # 链路本地
'172.16.0.0/12', # 私有网络
'192.0.0.0/24', # 保留地址（IANA）
'192.0.2.0/24', # TEST-NET-1
'192.88.99.0/24', # 6to4 中继
'192.168.0.0/16', # 私有网络
'198.18.0.0/15', # 网络基准测试
'198.51.100.0/24', # TEST-NET-2
'203.0.113.0/24', # TEST-NET-3
# 连续地址直到 IP 结束，特殊处理
# '224.0.0.0/4', #组播地址（D类）
# '240.0.0.0/4', #保留地址（E类）
)
keeplist = []
for iprange in keeprange:
ip, mask = iprange.split('/')
keeplist.append((utils.ip_string_to_num(ip), 32 - int(mask)))
mask_dict = dict((str(2 ** i), i) for i in range(8, 25))
def int2bytes2(n, pack=struct.pack):
'''将整数转换为大端序字节'''
return pack('>H', n)
# return bytes(map(lambda b: (-1 >> b & 255), (8, 0)))
def int2bytes4(n, pack=struct.pack):
'''将整数转换为大端序字节'''
return pack('>I', n)
# return bytes(map(lambda b: (n >> b & 255), (24, 16, 8, 0)))
def bytes2int(s):
nchars = len(s)
# string to int or long. Type depends on nchars
x = sum(ord(s[byte]) << 8 * (nchars - byte - 1) for byte in range(nchars))
return x
# +---------+
# | 4 bytes | <- data length
# +---------------+
# | 224 * 4 bytes | <- first ip number index
# +---------------+
# | 2n * 4 bytes | <- cn ip ranges data
# +------------------------+
# | b'end' and update info | <- end verify
# +------------------------+
lastip_s = 0
lastip_e = 0
index = {}
index_n = 0
index_fip = -1
offset = 0
padding = b'\xff\xff'
update = ""
iplist = []
fdi = open(self.cn_ipv4_range,"r")
for line in fdi.readlines():
lp = line.split()
iplist.append((utils.ip_string_to_num(lp[0]), mask_dict[lp[1]]))
iplist.extend(keeplist)
# 排序，不然无法处理
iplist.sort(key=lambda x: x[0])
# 随便算一下
buffering = len(iplist) * 8 + 224 * 4 + 64 + 4
buffer = bytearray(buffering)
for ip, mask in iplist:
ip_s = ip >> mask << mask
ip_e = (ip >> mask) + 1 << mask
# 判断连续
if ip_s <= lastip_e:
# 判断覆盖
if ip_e > lastip_e:
lastip_e = ip_e
continue
# 排除初始值
if lastip_e:
# 一段范围分为包含和排除
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
# 一个索引分为开始和结束
fip = ord(lastip_s[0]) * 2
if fip != index_fip:
# 前一个索引结束，序数多 1
# 避免无法搜索从当前索引结尾地址到下个索引开始地址
index[index_fip + 1] = index_b = int2bytes2(index_n)
# 当前索引开始
index[fip] = index_b
index_fip = fip
index_n += 2
offset += 8
lastip_s = ip_s
lastip_e = ip_e
# 添加最后一段范围
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
fip = ord(lastip_s[0]) * 2
if fip != index_fip:
index[index_fip + 1] = index_b = int2bytes2(index_n)
index[fip] = index_b
index_n += 2
offset += 8
# 添加最后一个结束索引
index[fip + 1] = int2bytes2(index_n)
# 写入文件
fd = open(self.cn_ipdb, 'wb', buffering)
fd.write(int2bytes4(offset))
for i in xrange(224 * 2):
fd.write(index.get(i, padding))
fd.write(buffer[:offset])
fd.write('endCN IP from ')
fd.write(update.encode(u'ascii'))
count = int(index_n // 2)
fd.write(', range count: %d' % count)
fd.close()
xlog.debug('include IP range number：%s' % count)
xlog.debug('save to file:%s' % self.cn_ipdb)
