def download(req):
#显式加载 CA，确保正常使用
global context
if context is None:
import ssl
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
context.verify_mode = ssl.CERT_REQUIRED
context.set_ciphers(ssl._RESTRICTED_SERVER_CIPHERS)
context.load_verify_locations(ca1)
context.load_verify_locations(ca2)
retry_delay = 2
timeout = 8
l = 0
while l is 0:
fd = None
try:
fd = urllib.request.urlopen(req, timeout=timeout, context=context)
l = int(fd.headers.get('Content-Length', 0))
except:
pass
if l is 0:
if fd:
fd.close()
log('链接直连 IP 库网址失败，%d 秒后重试' % retry_delay)
from time import sleep
sleep(retry_delay)
return fd, l
def download_both_cniplist_as_db(ipdb):
log('开始下载 APNIC 和 17mon IP')
global update
_iplist = download_cniplist(ipdb, parse_apnic_cniplist)
_update = update
iplist = download_cniplist(ipdb, parse_17mon_cniplist)
iplist.extend(_iplist)
update = '%s and %s' % (_update, update)
save_iplist_as_db(ipdb, iplist)
log('APNIC 和 17mon IP 已保存完毕')
def download_cniplist(ipdb, parse_cniplist):
#支持断点续传
global Req_APNIC, Req_17MON, update
if parse_cniplist is parse_apnic_cniplist:
if Req_APNIC is None:
Req_APNIC = req = urllib.request.Request(Url_APNIC)
update = None
name = 'APNIC'
elif parse_cniplist is parse_17mon_cniplist:
if Req_17MON is None:
Req_17MON = req = urllib.request.Request(Url_17MON)
import time
#设定为当月第一天
update = '17mon-' + time.strftime('%Y%m01', time.localtime(
time.time()))
name = '17mon'
req.remove_header('Range')
read = 0
l = None
while read != l:
fd, _l = download(req)
if l is None:
l = _l
iplist, _read = parse_cniplist(fd)
if _read is None:
read = l
else:
read += _read
fd.close()
#下载失败续传
if read != l:
#往回跳过可能的缺损条目
read = max(read - 100, 0)
req.remove_header('Range')
req.add_header('Range', 'bytes=%d-' % read)
log('%s IP 下载中断，续传：%d/%d' % (name, read, l))
log(name + ' IP 下载完毕')
return iplist
def save_iplist_as_db(ipdb, iplist):
# +---------+
# | 4 bytes | <- data length
# +---------------+
# | 224 * 4 bytes | <- first ip number index
# +---------------+
# | 2n * 4 bytes | <- cn ip ranges data
# +------------------------+
# | b'end' and update info | <- end verify
# +------------------------+
lastip_s = 0
lastip_e = 0
index = g_index
index_n = 0
index_fip = -1
offset = 0
iplist.extend(keeplist)
#排序，不然无法处理
iplist.sort(key=lambda x: x[0])
#随便算一下
buffering = len(iplist) * 8 + 224 * 4 + 64 + 4
buffer = bytearray(buffering)
for ip, mask in iplist:
ip_s = ip >> mask << mask
ip_e = (ip >> mask) + 1 << mask
#判断连续
if ip_s <= lastip_e:
#判断覆盖
if ip_e > lastip_e:
lastip_e = ip_e
continue
#排除初始值
if lastip_e:
#一段范围分为包含和排除
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
#一个索引分为开始和结束
fip = lastip_s[0] * 2
if fip != index_fip:
#前一个索引结束，序数多 1
#避免无法搜索从当前索引结尾地址到下个索引开始地址
index[index_fip + 1] = index_b = int2bytes2(index_n)
#当前索引开始
index[fip] = index_b
index_fip = fip
index_n += 2
offset += 8
lastip_s = ip_s
lastip_e = ip_e
#添加最后一段范围
buffer[offset:] = lastip_s = int2bytes4(lastip_s)
buffer[offset + 4:] = int2bytes4(lastip_e)
fip = lastip_s[0] * 2
if fip != index_fip:
index[index_fip + 1] = index_b = int2bytes2(index_n)
index[fip] = index_b
index_n += 2
offset += 8
#添加最后一个结束索引
index[fip + 1] = int2bytes2(index_n)
#写入文件
fd = open(ipdb, 'wb', buffering)
fd.write(int2bytes4(offset))
padding = g_padding
for i in range(224 * 2):
fd.write(index.get(i, padding))
fd.write(buffer[:offset])
fd.write(b'endCN IP from ')
fd.write(update.encode('ascii'))
fd.write(b', range count: ')
count = str(index_n // 2)
fd.write(count.encode('ascii'))
fd.close()
#清空缓存
g_iplist_apnic.clear()
g_iplist_17mon.clear()
g_index.clear()
log('更新信息：%s' % update)
log('包含 IP 范围条目数：%s' % count)
log('保存地址：%s' % ipdb)
def test(ipdb):
global update
update = 'keep IP test'
save_iplist_as_db(ipdb, [])
log('keeep IP 已保存完毕')
def download_17mon_cniplist_as_db(ipdb):
log('开始下载 17mon IP')
iplist = download_cniplist(ipdb, parse_17mon_cniplist)
save_iplist_as_db(ipdb, iplist)
log('17mon IP 已保存完毕')
def download_apnic_cniplist_as_db(ipdb):
log('开始下载 APNIC IP')
iplist = download_cniplist(ipdb, parse_apnic_cniplist)
save_iplist_as_db(ipdb, iplist)
log('APNIC IP 已保存完毕')