def queryIpWithUpdate(self, ip, masklen=24):
if not ip in self.previousSinaIpSet:
self.rnode = self.rtree.search_best(ip)
if not self.rnode:#this ip is not in the prefix table
jsonData = query_ip(ip)
if jsonData:
start = jsonData.get("start","")
if start:#we have the sina data:
end = jsonData["end"]
iprange = IPRange(start, end)
self.previousSinaIpSet = IPSet(iprange.cidrs())
for net_tuple in split_network_from_start_to_end(start, end):
self.addPrefix(net_tuple[0], net_tuple[1])
for k in ipRadixDB.recordKeys:
if k == "ip":
self.rnode.data[k] = jsonData.get(k,ip)
else:
self.rnode.data[k] = jsonData[k]
else:
self.addPrefix(ip, masklen)
for k in ipRadixDB.recordKeys:
self.rnode.data[k] = jsonData[k]
else:
if self.rnode.prefixlen < 24 or not self.rnode.data.get("country",""):#only the prefix is bigger than x/24 network
jsonData = query_ip(ip)#maybe the prefix is too large, we need to substrac the prefix
if jsonData:
jsonData["ip"] = ip
self.substractPrefix(self.rnode, jsonData)
def scan_cn_ip():
rtree = ipRadixDB()
all_done = True
f = open("input/delegated-apnic-latest","r")
count = 0
try:
for line in f.readlines():
if count % 100 == 0:
logger.info("==============================>[%s]", str(count))
count += 1
#l = line.decode("utf-8")
params = line.split("|")
if len(params) >=4 and params[1] == 'CN' and params[2] == "ipv4" and params[3] != "*":
network = params[3]
prefixlen = 32 - int(math.log(int(params[4]), 2))
prefix = network + '/' + str(prefixlen)
if int(prefixlen) > 24:
networks = IPNetwork(network+'/'+'24')
subnetworks = networks.subnet(24)
else:
networks = IPNetwork(prefix)
subnetworks = networks.subnet(24)
for sub in subnetworks:
network, prefixlen = str(sub).split("/")
ip = generate_random_ip(network, int(prefixlen))
rtree.addPrefix(network, int(prefixlen))
jsonData = query_ip(ip)
data_key = ["country", "province", "city", "isp"]
node = rtree.rnode
for key in data_key:
node.data[key] = jsonData.get(key,"")
node.data["ip"] = ip
logger.info("%s [%s] [%s] [%s]" % (ip, prefix, str(sub), node.data["province"]))
except Exception, e:
logger.error(e)
all_done = False
def scan_fn_ip():
country_code = {}
for line in open('input/country_code', 'r'):
code, name = line.split(" ")
country_code[code] = name.strip().decode("utf-8")
logger.info(code + ' ' + country_code[code])
rtree = ipRadixDB()
ip_area_list = ["input/delegated-arin-latest", "input/delegated-ripencc-latest", "input/delegated-lacnic-latest", "input/delegated-afrinic-latest", "input/delegated-apnic-latest"]
dft = defaultdict(list)
availableIPs = []
for f in ip_area_list:
seed_file = open(f,'r')
for l in seed_file.readlines():
params = l.split('|')
if len(params) >= 4 and params[2] == "ipv4" and params[3] != "*" and params[1] != "CN":
startIP = params[3]
endIP = ip_integer_to_string(ip_integer_from_string(startIP) + int(params[4]) - 1)
logger.info(startIP + ' ' + endIP + ' ' + params[4])
iprange = IPRange(startIP, endIP)
if params[1] == '':
availableIPs += map(str, iprange.cidrs())
else:
dft[params[1]] += map(str, iprange.cidrs())
for key in dft:
prefix = dft[key][-1]
network,masklen = prefix.split('/')
masklen = int(masklen)
ip = generate_random_ip(network,masklen)
ipset = IPSet(dft[key])
for prefix in ipset.iter_cidrs():
network,masklen = str(prefix).split('/')
masklen = int(masklen)
rtree.addPrefix(network,masklen)
data = rtree.rnode.data
country = country_code[key]
logger.info(str(prefix) + ' ' + country)
data['country'] = country #jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = prefix.size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
for prefix in availableIPs:
network,masklen = prefix.split("/")
masklen = int(masklen)
ip = generate_random_ip(network,masklen)
jsonData = None;
while jsonData == None:
try:
jsonData = query_ip(ip)
except Exception, e:
logger.error(e)
time.sleep(0.5)
rtree.addPrefix(network,masklen)
data = rtree.rnode.data
data['country'] = jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = IPNetwork(prefix).size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
logger.info(prefix + ' ' + data['country'])
def scan_fn_ip():
country_code = {}
for line in open('input/country_code', 'r'):
code, name = line.split(" ")
country_code[code] = name.strip().decode("utf-8")
logger.info(code + ' ' + country_code[code])
rtree = ipRadixDB()
ip_area_list = [
"input/delegated-arin-latest", "input/delegated-ripencc-latest",
"input/delegated-lacnic-latest", "input/delegated-afrinic-latest",
"input/delegated-apnic-latest"
]
dft = defaultdict(list)
availableIPs = []
for f in ip_area_list:
seed_file = open(f, 'r')
for l in seed_file.readlines():
params = l.split('|')
if len(params) >= 4 and params[
2] == "ipv4" and params[3] != "*" and params[1] != "CN":
startIP = params[3]
endIP = ip_integer_to_string(
ip_integer_from_string(startIP) + int(params[4]) - 1)
logger.info(startIP + ' ' + endIP + ' ' + params[4])
iprange = IPRange(startIP, endIP)
if params[1] == '':
availableIPs += map(str, iprange.cidrs())
else:
dft[params[1]] += map(str, iprange.cidrs())
for key in dft:
prefix = dft[key][-1]
network, masklen = prefix.split('/')
masklen = int(masklen)
ip = generate_random_ip(network, masklen)
ipset = IPSet(dft[key])
for prefix in ipset.iter_cidrs():
network, masklen = str(prefix).split('/')
masklen = int(masklen)
rtree.addPrefix(network, masklen)
data = rtree.rnode.data
country = country_code[key]
logger.info(str(prefix) + ' ' + country)
data['country'] = country #jsonData.get('country','')
data['ip'] = ip
data['ip_amount'] = prefix.size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
for prefix in availableIPs:
network, masklen = prefix.split("/")
masklen = int(masklen)
ip = generate_random_ip(network, masklen)
jsonData = None
while jsonData == None:
try:
jsonData = query_ip(ip)
except Exception, e:
logger.error(e)
time.sleep(0.5)
rtree.addPrefix(network, masklen)
data = rtree.rnode.data
data['country'] = jsonData.get('country', '')
data['ip'] = ip
data['ip_amount'] = IPNetwork(prefix).size
data['province'] = ''
data['city'] = ''
data['isp'] = ''
logger.info(prefix + ' ' + data['country'])