def load_ec2_range(self):
fn = os.path.join(current_path, "ec2_iprange.txt")
with open(fn, "r") as fd:
ipr = json.load(fd)
for ipd in ipr["prefixes"]:
try:
if ipd["region"] != "us-east-1":
continue
if "ip_prefix" not in ipd:
continue
if ipd["service"] != "EC2":
continue
ipp = ipd["ip_prefix"]
# xlog.debug("range:%s", ipp)
begin, end = ip_utils.split_ip(ipp)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
self.ip_range_list.append([nbegin, nend])
except:
continue
def load_ip_range(self):
logging.info("load ip range file:%s", self.range_file)
fd = open(self.range_file, "r")
if not fd:
print "open ip_range.txt fail."
exit()
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
logging.error("load ip range:%s", line)
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
fd.close()
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def parse_range_string(input_lines):
ip_range_list = []
ip_lines_list = re.split("\r|\n", input_lines)
for raw_line in ip_lines_list:
raw_s = raw_line.split("#")
context_line = raw_s[0]
context_line = context_line.replace(' ', '')
ips = re.split(",|\|", context_line)
for line in ips:
if len(line) == 0:
#print "non line:", line
continue
begin, end = ip_utils.split_ip(line)
if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(end) == 0:
print("ip format is error,line:%s, begin: %s,end: %s" % (line, begin, end))
continue
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
ip_range_list.append([nbegin,nend])
#print begin, end
ip_range_list.sort()
return ip_range_list
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def parse_range_string(input_lines):
ip_range_list = []
ip_lines_list = re.split("\r|\n", input_lines)
for raw_line in ip_lines_list:
raw_s = raw_line.split("#")
context_line = raw_s[0]
context_line = context_line.replace(' ', '')
ips = re.split(",|\|", context_line)
for line in ips:
if len(line) == 0:
#print "non line:", line
continue
begin, end = ip_utils.split_ip(line)
if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(
end) == 0:
PRINT("ip format is error,line:%s, begin: %s,end: %s" %
(line, begin, end))
continue
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
ip_range_list.append([nbegin, nend])
#print begin, end
ip_range_list.sort()
return ip_range_list
def test_load():
ip_str = []
print("\nBegin test load googleip.txt")
fd = open("googleip.txt", "r")
if not fd:
print "open googleip.txt fail."
exit()
amount = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
ip_begin_str = begin.strip().split('.')
ip_end_str = end.strip().split('.')
if ip_begin_str[3] == '0': ip_begin_str[3] = '1'
if ip_end_str[3] == '255': ip_end_str[3] = '254'
str_1 = (int(ip_end_str[0]) - int(ip_begin_str[0])) * 16646144
str_2 = (int(ip_end_str[1]) - int(ip_begin_str[1])) * 65024
str_3 = (int(ip_end_str[2]) - int(ip_begin_str[2])) * 254
str_4 = int(ip_end_str[3]) - int(ip_begin_str[3]) + 1
num = str_1 + str_2 + str_3 + str_4
amount += num
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
fd.close()
print "amount ip:", amount
def load_ip_range(self):
fd = open("ip_range.txt", "r")
if not fd:
print "open ip_range.txt fail."
exit()
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append([nbegin, nend])
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
fd.close()
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.candidate_amount_ip = 0
lines = []
content = self.load_range_content()
raw_lines = content.splitlines()
for ip_line in raw_lines:
ip_line = ip_line.replace(' ', '')
if len(ip_line) == 0 or ip_line[0] == '#':
continue
ips = ip_line.split("|")
for ip in ips:
if len(ip) == 0:
continue
lines.append(ip)
if config.USE_IPV6:
self.ipv6_list = lines
return
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
num = nend - nbegin
self.candidate_amount_ip += num
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.candidate_amount_ip = 0
lines = []
content = self.load_range_content()
raw_lines = content.splitlines()
for ip_line in raw_lines:
ip_line = ip_line.replace(' ', '')
if len(ip_line) == 0 or ip_line[0] == '#':
continue
ips = ip_line.split("|")
for ip in ips:
if len(ip) == 0:
continue
lines.append(ip)
if config.USE_IPV6:
self.ipv6_list = lines
return
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append([nbegin, nend])
num = nend - nbegin
self.candidate_amount_ip += num
def test_load():
fd = open("ip_range.txt", "r")
if not fd:
print "open ip_range.txt fail."
exit()
amount = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
num = nend - nbegin
amount += num
print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
fd.close()
print "amount:", amount
def test_load():
fd = open("ip_range.txt", "r")
if not fd:
print "open ip_range.txt fail."
exit()
amount = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == "#":
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
num = nend - nbegin
amount += num
print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
fd.close()
print "amount:", amount
def test_load():
print("\nBegin test load googleip.txt")
fd = open("googleip.txt", "r")
if not fd:
print "open googleip.txt fail."
exit()
amount = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
num = nend - nbegin
amount += num
print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
fd.close()
print "amount ip:", amount
def test_load():
print("Begin test load ip_range.txt\n")
file_name = os.path.join(config.DATA_PATH, "ip_range.txt")
fd = open(file_name, "r")
if not fd:
print "open ip_range.txt fail."
exit()
amount = 0
for line in fd.readlines():
if len(line) == 0 or line[0] == '#':
continue
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
num = nend - nbegin
amount += num
print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
fd.close()
print "amount ip:", amount
def generate_bin(self):
xlog.info("generating binary ip pool file.")
rfd = open(self.txt_ip_fn, "rt")
wfd = open(self.bin_ip_fn, "wb")
num = 0
for line in rfd.readlines():
ip = line
try:
ip_num = ip_utils.ip_string_to_num(ip)
except Exception as e:
xlog.warn("ip %s not valid in %s", ip, self.txt_ip_fn)
continue
ip_bin = struct.pack("<I", ip_num)
wfd.write(ip_bin)
num += 1
rfd.close()
wfd.close()
xlog.info("finished generate binary ip pool file, num:%d", num)
def generate_bin(self):
xlog.info("generating binary ip pool file.")
rfd = open(self.txt_ip_fn, "rt")
wfd = open(self.bin_ip_fn, "wb")
num = 0
for line in rfd.readlines():
ip = line
try:
ip_num = ip_utils.ip_string_to_num(ip)
except Exception as e:
xlog.warn("ip %s not valid in %s", ip, self.txt_ip_fn)
continue
ip_bin = struct.pack("<I", ip_num)
wfd.write(ip_bin)
num += 1
rfd.close()
wfd.close()
xlog.info("finished generate binary ip pool file, num:%d", num)
def merge_ip_range():
ip_range_list = []
ip_lines_list = re.split("\r|\n", ip_str_list)
for iplines in ip_lines_list:
if len(iplines) == 0 or iplines[0] == "#":
# print "non:", iplines
continue
ips = re.split(",|\|", iplines)
for line in ips:
if len(line) == 0 or line[0] == "#":
# print "non line:", line
continue
begin, end = ip_utils.split_ip(line)
if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(end) == 0:
PRINT("ip format is error,line:%s, begin: %s,end: %s" % (line, begin, end))
continue
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
ip_range_list.append([nbegin, nend])
# print begin, end
ip_range_list.sort()
# merge range
ip_range_list_2 = []
range_num = len(ip_range_list)
last_begin = ip_range_list[0][0]
last_end = ip_range_list[0][1]
for i in range(1, range_num - 1):
ip_range = ip_range_list[i]
begin = ip_range[0]
end = ip_range[1]
# print "now:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
if begin > last_end + 2:
# print "add:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
ip_range_list_2.append([last_begin, last_end])
last_begin = begin
last_end = end
else:
print "merge:", ip_utils.ip_num_to_string(last_begin), ip_utils.ip_num_to_string(
last_end
), ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
if end > last_end:
last_end = end
ip_range_list_2.append([last_begin, last_end])
for ip_range in ip_range_list_2:
begin = ip_range[0]
end = ip_range[1]
print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
# write out
fd = open("ip_range.txt", "w")
for ip_range in ip_range_list_2:
begin = ip_range[0]
end = ip_range[1]
# print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
fd.write(ip_utils.ip_num_to_string(begin) + "-" + ip_utils.ip_num_to_string(end) + "\n")
fd.close()
def report_bad_ip(ip_str):
ip_bin = ip_utils.ip_string_to_num(ip_str)
ip_bin = ip_bin & 0xffffff00
ip_mask = ip_utils.ip_num_to_string(ip_bin)
return ip_mask
def report_bad_ip(ip_str):
ip_bin = ip_utils.ip_string_to_num(ip_str)
ip_bin = ip_bin & 0xffffff00
ip_mask = ip_utils.ip_num_to_string(ip_bin)
return ip_mask
def merge_ip_range():
ip_range_list = []
ip_lines_list = re.split("\r|\n", ip_str_list)
for iplines in ip_lines_list:
if len(iplines) == 0 or iplines[0] == '#':
#print "non:", iplines
continue
ips = re.split(",|\|", iplines)
for line in ips:
if len(line) == 0 or line[0] == '#':
#print "non line:", line
continue
begin, end = ip_utils.split_ip(line)
if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(
end) == 0:
PRINT("ip format is error,line:%s, begin: %s,end: %s" %
(line, begin, end))
continue
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
ip_range_list.append([nbegin, nend])
#print begin, end
ip_range_list.sort()
# merge range
ip_range_list_2 = []
range_num = len(ip_range_list)
last_begin = ip_range_list[0][0]
last_end = ip_range_list[0][1]
for i in range(1, range_num - 1):
ip_range = ip_range_list[i]
begin = ip_range[0]
end = ip_range[1]
#print "now:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
if begin > last_end + 2:
#print "add:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
ip_range_list_2.append([last_begin, last_end])
last_begin = begin
last_end = end
else:
print "merge:", ip_utils.ip_num_to_string(
last_begin), ip_utils.ip_num_to_string(
last_end), ip_utils.ip_num_to_string(
begin), ip_utils.ip_num_to_string(end)
if end > last_end:
last_end = end
ip_range_list_2.append([last_begin, last_end])
for ip_range in ip_range_list_2:
begin = ip_range[0]
end = ip_range[1]
print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
# write out
fd = open("ip_range.txt", "w")
for ip_range in ip_range_list_2:
begin = ip_range[0]
end = ip_range[1]
#print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
fd.write(
ip_utils.ip_num_to_string(begin) + "-" +
ip_utils.ip_num_to_string(end) + "\n")
fd.close()
