def load_ip_range(self): self.ip_range_map = {} self.ip_range_list = [] self.ip_range_index = [] self.candidate_amount_ip = 0 content = self.load_range_content() lines = content.splitlines() for line in lines: if len(line) == 0 or line[0] == '#': continue try: begin, end = utils.split_ip(line) nbegin = utils.ip_string_to_num(begin) nend = utils.ip_string_to_num(end) if not nbegin or not nend or nend < nbegin: self.logger.warn("load ip range:%s fail", line) continue except Exception as e: self.logger.exception("load ip range:%s fail:%r", line, e) continue self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend] self.ip_range_list.append( [nbegin, nend] ) self.ip_range_index.append(self.candidate_amount_ip) num = nend - nbegin self.candidate_amount_ip += num # print utils.ip_num_to_string(nbegin), utils.ip_num_to_string(nend), num self.ip_range_index.sort()
def load_ip_range(self): self.ip_range_map = {} self.ip_range_list = [] self.ip_range_index = [] self.candidate_amount_ip = 0 content = self.load_range_content() lines = content.splitlines() for line in lines: if len(line) == 0 or line[0] == '#': continue try: begin, end = utils.split_ip(line) nbegin = utils.ip_string_to_num(begin) nend = utils.ip_string_to_num(end) if not nbegin or not nend or nend < nbegin: self.logger.warn("load ip range:%s fail", line) continue except Exception as e: self.logger.exception("load ip range:%s fail:%r", line, e) continue self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend] self.ip_range_list.append([nbegin, nend]) self.ip_range_index.append(self.candidate_amount_ip) num = nend - nbegin self.candidate_amount_ip += num # print utils.ip_num_to_string(nbegin), utils.ip_num_to_string(nend), num self.ip_range_index.sort()
def generate_bin(self): self.logger.info("generating binary ip pool file.") rfd = open(self.source_txt_fn, "rt") wfd = open(self.dest_bin_fn, "wb") num = 0 for line in rfd.readlines(): ip = line try: ip_num = utils.ip_string_to_num(ip) except Exception as e: self.logger.warn("ip %s not valid in %s", ip, self.source_txt_fn) continue ip_bin = struct.pack("<I", ip_num) wfd.write(ip_bin) num += 1 rfd.close() wfd.close() self.logger.info("finished generate binary ip pool file, num:%d", num)
def generate_db(self): keeprange = ( '0.0.0.0/8', # 本地网络 '10.0.0.0/8', # 私有网络 '100.64.0.0/10', # 地址共享(运营商 NAT) '127.0.0.0/8', # 环回地址 '169.254.0.0/16', # 链路本地 '172.16.0.0/12', # 私有网络 '192.0.0.0/24', # 保留地址(IANA) '192.0.2.0/24', # TEST-NET-1 '192.88.99.0/24', # 6to4 中继 '192.168.0.0/16', # 私有网络 '198.18.0.0/15', # 网络基准测试 '198.51.100.0/24', # TEST-NET-2 '203.0.113.0/24', # TEST-NET-3 # 连续地址直到 IP 结束,特殊处理 # '224.0.0.0/4', #组播地址(D类) # '240.0.0.0/4', #保留地址(E类) ) keeplist = [] for iprange in keeprange: ip, mask = iprange.split('/') keeplist.append((utils.ip_string_to_num(ip), 32 - int(mask))) mask_dict = dict((str(2**i), i) for i in range(8, 25)) def int2bytes2(n, pack=struct.pack): '''将整数转换为大端序字节''' return pack('>H', n) # return bytes(map(lambda b: (-1 >> b & 255), (8, 0))) def int2bytes4(n, pack=struct.pack): '''将整数转换为大端序字节''' return pack('>I', n) # return bytes(map(lambda b: (n >> b & 255), (24, 16, 8, 0))) def bytes2int(s): nchars = len(s) # string to int or long. Type depends on nchars x = sum( ord(s[byte]) << 8 * (nchars - byte - 1) for byte in range(nchars)) return x # +---------+ # | 4 bytes | <- data length # +---------------+ # | 224 * 4 bytes | <- first ip number index # +---------------+ # | 2n * 4 bytes | <- cn ip ranges data # +------------------------+ # | b'end' and update info | <- end verify # +------------------------+ lastip_s = 0 lastip_e = 0 index = {} index_n = 0 index_fip = -1 offset = 0 padding = b'\xff\xff' update = "" iplist = [] fdi = open(self.cn_ipv4_range, "r") for line in fdi.readlines(): lp = line.split() iplist.append((utils.ip_string_to_num(lp[0]), mask_dict[lp[1]])) iplist.extend(keeplist) # 排序,不然无法处理 iplist.sort(key=lambda x: x[0]) # 随便算一下 buffering = len(iplist) * 8 + 224 * 4 + 64 + 4 buffer = bytearray(buffering) for ip, mask in iplist: ip_s = ip >> mask << mask ip_e = (ip >> mask) + 1 << mask # 判断连续 if ip_s <= lastip_e: # 判断覆盖 if ip_e > lastip_e: lastip_e = ip_e continue # 排除初始值 if lastip_e: # 一段范围分为包含和排除 buffer[offset:] = lastip_s = int2bytes4(lastip_s) buffer[offset + 4:] = int2bytes4(lastip_e) # 一个索引分为开始和结束 fip = ord(lastip_s[0:1]) * 2 if fip != index_fip: # 前一个索引结束,序数多 1 # 避免无法搜索从当前索引结尾地址到下个索引开始地址 index[index_fip + 1] = index_b = int2bytes2(index_n) # 当前索引开始 index[fip] = index_b index_fip = fip index_n += 2 offset += 8 lastip_s = ip_s lastip_e = ip_e # 添加最后一段范围 buffer[offset:] = lastip_s = int2bytes4(lastip_s) buffer[offset + 4:] = int2bytes4(lastip_e) fip = ord(lastip_s[0:1]) * 2 if fip != index_fip: index[index_fip + 1] = index_b = int2bytes2(index_n) index[fip] = index_b index_n += 2 offset += 8 # 添加最后一个结束索引 index[fip + 1] = int2bytes2(index_n) # 写入文件 fd = open(self.cn_ipdb, 'wb', buffering) fd.write(int2bytes4(offset)) for i in range(224 * 2): fd.write(index.get(i, padding)) fd.write(buffer[:offset]) fd.write(b'endCN IP from ') fd.write(update.encode('ascii')) count = int(index_n // 2) fd.write(b', range count: %d' % count) fd.close() xlog.debug('include IP range number:%s' % count) xlog.debug('save to file:%s' % self.cn_ipdb)
def generate_db(self): keeprange = ( '0.0.0.0/8', # 本地网络 '10.0.0.0/8', # 私有网络 '100.64.0.0/10', # 地址共享(运营商 NAT) '127.0.0.0/8', # 环回地址 '169.254.0.0/16', # 链路本地 '172.16.0.0/12', # 私有网络 '192.0.0.0/24', # 保留地址(IANA) '192.0.2.0/24', # TEST-NET-1 '192.88.99.0/24', # 6to4 中继 '192.168.0.0/16', # 私有网络 '198.18.0.0/15', # 网络基准测试 '198.51.100.0/24', # TEST-NET-2 '203.0.113.0/24', # TEST-NET-3 # 连续地址直到 IP 结束,特殊处理 # '224.0.0.0/4', #组播地址(D类) # '240.0.0.0/4', #保留地址(E类) ) keeplist = [] for iprange in keeprange: ip, mask = iprange.split('/') keeplist.append((utils.ip_string_to_num(ip), 32 - int(mask))) mask_dict = dict((str(2 ** i), i) for i in range(8, 25)) def int2bytes2(n, pack=struct.pack): '''将整数转换为大端序字节''' return pack('>H', n) # return bytes(map(lambda b: (-1 >> b & 255), (8, 0))) def int2bytes4(n, pack=struct.pack): '''将整数转换为大端序字节''' return pack('>I', n) # return bytes(map(lambda b: (n >> b & 255), (24, 16, 8, 0))) def bytes2int(s): nchars = len(s) # string to int or long. Type depends on nchars x = sum(ord(s[byte]) << 8 * (nchars - byte - 1) for byte in range(nchars)) return x # +---------+ # | 4 bytes | <- data length # +---------------+ # | 224 * 4 bytes | <- first ip number index # +---------------+ # | 2n * 4 bytes | <- cn ip ranges data # +------------------------+ # | b'end' and update info | <- end verify # +------------------------+ lastip_s = 0 lastip_e = 0 index = {} index_n = 0 index_fip = -1 offset = 0 padding = b'\xff\xff' update = "" iplist = [] fdi = open(self.cn_ipv4_range,"r") for line in fdi.readlines(): lp = line.split() iplist.append((utils.ip_string_to_num(lp[0]), mask_dict[lp[1]])) iplist.extend(keeplist) # 排序,不然无法处理 iplist.sort(key=lambda x: x[0]) # 随便算一下 buffering = len(iplist) * 8 + 224 * 4 + 64 + 4 buffer = bytearray(buffering) for ip, mask in iplist: ip_s = ip >> mask << mask ip_e = (ip >> mask) + 1 << mask # 判断连续 if ip_s <= lastip_e: # 判断覆盖 if ip_e > lastip_e: lastip_e = ip_e continue # 排除初始值 if lastip_e: # 一段范围分为包含和排除 buffer[offset:] = lastip_s = int2bytes4(lastip_s) buffer[offset + 4:] = int2bytes4(lastip_e) # 一个索引分为开始和结束 fip = ord(lastip_s[0]) * 2 if fip != index_fip: # 前一个索引结束,序数多 1 # 避免无法搜索从当前索引结尾地址到下个索引开始地址 index[index_fip + 1] = index_b = int2bytes2(index_n) # 当前索引开始 index[fip] = index_b index_fip = fip index_n += 2 offset += 8 lastip_s = ip_s lastip_e = ip_e # 添加最后一段范围 buffer[offset:] = lastip_s = int2bytes4(lastip_s) buffer[offset + 4:] = int2bytes4(lastip_e) fip = ord(lastip_s[0]) * 2 if fip != index_fip: index[index_fip + 1] = index_b = int2bytes2(index_n) index[fip] = index_b index_n += 2 offset += 8 # 添加最后一个结束索引 index[fip + 1] = int2bytes2(index_n) # 写入文件 fd = open(self.cn_ipdb, 'wb', buffering) fd.write(int2bytes4(offset)) for i in xrange(224 * 2): fd.write(index.get(i, padding)) fd.write(buffer[:offset]) fd.write('endCN IP from ') fd.write(update.encode(u'ascii')) count = int(index_n // 2) fd.write(', range count: %d' % count) fd.close() xlog.debug('include IP range number:%s' % count) xlog.debug('save to file:%s' % self.cn_ipdb)