def whitelist_filter(fullmatch, segmentmatch, lpmmatch, lpmfullmatch,
whitelist):
# treat whitelist data
white_full, white_segment, white_subnet = separate_ip(whitelist)
# filter each match result
lpm.init()
for sn in white_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): # 略过私网
continue
# return 'False'
elif (netMask == 16):
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask == 24):
# /24处理
newip1 = []
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
else:
# netMask>16 and not in [16,23,24,25],save them
continue
fullmatch_set = set(fullmatch) - (set(fullmatch) & set(white_full))
matchlist = []
for ips in fullmatch_set:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
matchlist.append(ips)
fullmatch = list(fullmatch_set - set(matchlist))
#subnet
tmpmatch = []
if (len(lpmmatch) > 0):
for i in lpmmatch:
ips = i.keys()[0]
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (ips in white_full or lpm.search_ip(ip_es_num)):
tmpmatch.append(i)
for ii in tmpmatch:
lpmmatch.remove(ii)
return fullmatch, segmentmatch, lpmmatch, lpmfullmatch
)
},
LPM_ITEM_STATUS_MAP={
'LP0002': {
'tested': dict(origins=[''], unavailable=False, role='item_admin'),
'reserved': dict(origins=['', 'tested'], unavailable=False),
'shipped': dict(origins=['reserved'], unavailable=True, require_admin=False),
'obsolete': dict(origins=['', 'tested', 'reserved', 'shipped'],
unavailable=True, role='item_admin')
},
'default': {
'obsolete': dict(origins=[''], unavailable=True, role='item_admin')
},
},
LPM_EXT_UPDATE_FIELDS={
'default': set(),
},
)
lpm.init(app)
@app.route('/')
def main():
"""
main entry point, redirect to the items overview
"""
return redirect(url_for('items.overview'))
app.run()
def lpm_match(subnet_data, remain_data):
mylog = getlog()
lpm.init()
sn_gte24 = []
# ip_subnet = subnet.keys()
for sn in subnet_data:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
# if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12' or sn == '10.0.0.0/8'): # 略过私网
# continue
# return 'False'
if (netMask <= 8):
idx = pow(2, 8 - netMask) - 1
# print idx
ip_base = ip_num[0] & (255 - idx)
i = 0
while (i <= idx):
newip1 = []
ip_num[0] = ip_base + i
newip1.append(str(ip_num[0]))
newip1.append('*')
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
# print ipstr1
lpm.insert_rule(ipstr1)
i = i + 1
elif (netMask <= 16): #
idx = pow(2, 16 - netMask) - 1
# print idx
ip_base = ip_num[1] & (255 - idx)
i = 0
while (i <= idx):
newip1 = []
ip_num[1] = ip_base + i
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
# print ipstr1
lpm.insert_rule(ipstr1)
i = i + 1
elif (netMask <= 24):
idx = pow(2, 24 - netMask) - 1
# print idx
ip_base = ip_num[2] & (255 - idx)
i = 0
while (i <= idx):
newip1 = []
ip_num[2] = ip_base + i
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
# print ipstr1
lpm.insert_rule(ipstr1)
i = i + 1
elif (netMask > 24): # range match
sn_gte24.append(sn)
mylog.info('[LPM] gte24 size:%d' % len(sn_gte24))
# match
new_remain = []
for ips in remain_data:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
continue
else:
new_remain.append(ips)
return sn_gte24, new_remain
def subnet_lpm(subnet, es_ip):
lpm.init()
sndict = {}
fpath = parser_config.get_store_path()[1]
sn_lte16 = {}
ip_subnet = subnet.keys()
for sn in ip_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): #略过私网
continue
# return 'False'
elif (netMask < 16): #暂时不处理
sn_lte16[sn] = subnet[sn]
# return 'False'
elif (netMask == 16):
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask == 23):
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
newip2 = []
ip_num[2] = ip_num[2] & 0
newip2.append(str(ip_num[0]))
newip2.append(str(ip_num[1]))
newip2.append(str(ip_num[2]))
newip2.append('*')
ipstr2 = '.'.join(newip2)
lpm.insert_rule(ipstr2)
elif (netMask == 25 or netMask == 24):
#/25当/24处理
newip1 = []
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
else:
#netMask>16 and not in [16,3,24,25],save them
sndict[sn] = subnet[sn]
saveToJSON(sndict, fpath, "remain_subnet")
saveToJSON(sn_lte16, fpath, 'lte16_subnet')
#match
subnet_result = []
for ips in es_ip:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
subnet_result.append({ips: 'subnet_lpm'})
return subnet_result, sndict, sn_lte16
def setUp(self):
self.app = Flask("lpm_test")
self.app.config.update(
DEBUG=True,
SECRET_KEY=b"\xfeb\xba\x0eU\xb7\xd1\xdd\xe05k\x9eF%\xf93&r\xe3y\xe7\x9c\xac\x9a",
MONGO_DBNAME="lpm_test",
WTF_CSRF_ENABLED=False,
LPM_AUTH_SRC="simple",
LPM_AUTH_USERS={
"worker": dict(name="Worker", password="******", roles={"login", "component_edit"}, active=True),
"admin": dict(
name="Admin",
password="******",
roles={
"login",
"request_login",
"component_edit",
"component_admin",
"stock_admin",
"item_admin",
"db_debug",
},
active=True,
),
"viewer": dict(name="Viewer", password="******", roles={"login", "request_login"}, active=True),
"disabled": dict(
name="Disabled User", password="******", roles={"login", "component_admin"}, active=False
),
"ext": dict(
name="External Scripts", password="******", roles={"request_login", "component_admin"}, active=True
),
},
LPM_PARTNO_PREFIX="LP",
LPM_COMPONENT_CATEGORIES={"category1", "category2"},
LPM_COMPONENT_FILES_DIR="/tmp",
LPM_ITEM_VIEW_MAP={"TE0002": "TE0002.html", "TE0001a": "TE0001a.html", "TE0001b": "TE0001b.html"},
LPM_ITEM_IMPORT_MAP={
"TE0002": dict(required_fields=["param1", "param2", "param3", "param4"], integer_fields=["param2"]),
"TE0001a": dict(
required_fields=["param5", "param6", "param7", "param8"],
date_fields=["param5"],
integer_fields=["param2", "param6"],
floating_point_fields=["param7"],
boolean_fields=["param8"],
),
},
LPM_ITEM_STATUS_MAP={
"TE0002": {
"tested": dict(origins=[""], unavailable=False, role="item_admin"),
"reserved": dict(origins=["", "tested"], unavailable=False),
"shipped": dict(origins=["reserved"], unavailable=True, require_admin=False),
"obsolete": dict(
origins=["", "tested", "reserved", "shipped"], unavailable=True, role="item_admin"
),
},
"default": {"obsolete": dict(origins=[""], unavailable=True, role="item_admin")},
},
LPM_EXT_UPDATE_FIELDS={"TE0001a": {"key1", "key2"}, "default": set()},
)
lpm.init(self.app)
self.client = self.app.test_client()
def subnet_lpm(subnet, es_ip):
mylog = blacklist_tools.getlog()
lpm.init()
sndict = {}
fpath = parser_config.get_store_path()[1]
sn_lte16 = {}
lpmdict = {}
sn_gte24 = {}
ip_subnet = subnet.keys()
for sn in ip_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): #略过私网
continue
# return 'False'
elif (netMask < 16): #暂时不处理
sn_lte16[sn] = subnet[sn]
# return 'False'
elif (netMask == 16):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask >= 21 and netMask <= 24):
lpmdict[sn] = subnet[sn]
idx = pow(2, 24 - netMask) - 1
# print idx
ip_base = ip_num[2] & (255 - idx)
i = 0
while (i <= idx):
newip1 = []
ipstr1 = ''
ip_num[2] = ip_base + i
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
# print ipstr1
lpm.insert_rule(ipstr1)
i = i + 1
# elif(netMask==24):
# #/25当/24处理
# lpmdict[sn] = subnet[sn]
# newip1 = []
# newip1.append(str(ip_num[0]))
# newip1.append(str(ip_num[1]))
# newip1.append(str(ip_num[2]))
# newip1.append('*')
# ipstr1 = '.'.join(newip1)
# lpm.insert_rule(ipstr1)
elif (netMask > 24): # range match
sn_gte24[sn] = subnet[sn]
else:
#netMask>16 and netMask<21,save them
sndict[sn] = subnet[sn]
mylog.info('lpm data size: %d' % len(lpmdict))
mylog.info('remaining subnet size:%d' % len(sndict))
mylog.info('lte16 size:%d' % len(sn_lte16))
mylog.info('gte24 size:%d' % len(sn_gte24))
#save
snpath = getsavepath(fpath, 'remaining_subnet')
ltepath = getsavepath(fpath, 'lte16_subnet')
lpmpath = getsavepath(fpath, 'lpm_subnet_data')
gtepath = getsavepath(fpath, 'gte24_subnet')
if (sndict):
if (os.path.exists(snpath)):
newsndict = blacklist_tools.load_dict(snpath)
newsndict1 = dict(newsndict, **sndict) #merge
saveToJSON(newsndict1, fpath, "remaining_subnet")
else:
saveToJSON(sndict, fpath, "remaining_subnet")
if (sn_lte16):
if (os.path.exists(ltepath)):
newlte = blacklist_tools.load_dict(ltepath)
newlte16 = dict(newlte, **sn_lte16) #merge
saveToJSON(newlte16, fpath, 'lte16_subnet')
else:
saveToJSON(sn_lte16, fpath, 'lte16_subnet')
if (lpmdict):
if (os.path.exists(lpmpath)):
newlpmdict = blacklist_tools.load_dict(lpmpath)
newlpmdict1 = dict(newlpmdict, **lpmdict) #merge
saveToJSON(newlpmdict1, fpath, 'lpm_subnet_data')
else:
saveToJSON(lpmdict, fpath, 'lpm_subnet_data')
if (sn_gte24):
if (os.path.exists(gtepath)):
newlpmdict = blacklist_tools.load_dict(gtepath)
newlpmdict1 = dict(newlpmdict, **sn_gte24) #merge
saveToJSON(newlpmdict1, fpath, 'gte24_subnet')
else:
saveToJSON(sn_gte24, fpath, 'gte24_subnet')
sn_gte24 = dict(sn_gte24, **sndict) # merge
#match
subnet_result = []
for ips in es_ip:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
subnet_result.append({ips: 'subnet_lpm_match'})
return subnet_result, sndict, sn_lte16, sn_gte24
def subnet_lpm(subnet, es_ip):
mylog = blacklist_tools.getlog()
lpm.init()
sndict = {}
fpath = parser_config.get_store_path()[1]
sn_lte16 = {}
lpmdict = {}
ip_subnet = subnet.keys()
for sn in ip_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): #略过私网
continue
# return 'False'
elif (netMask < 16): #暂时不处理
sn_lte16[sn] = subnet[sn]
# return 'False'
elif (netMask == 16):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask == 23):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
newip2 = []
ip_num[2] = ip_num[2] & 254
newip2.append(str(ip_num[0]))
newip2.append(str(ip_num[1]))
newip2.append(str(ip_num[2]))
newip2.append('*')
ipstr2 = '.'.join(newip2)
lpm.insert_rule(ipstr2)
elif (netMask == 25 or netMask == 24):
#/25当/24处理
lpmdict[sn] = subnet[sn]
newip1 = []
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
else:
#netMask>16 and not in [16,23,24,25],save them
sndict[sn] = subnet[sn]
mylog.info('lpm data size: %d' % len(lpmdict))
#save
snpath = getsavepath(fpath, 'remain_subnet')
ltepath = getsavepath(fpath, 'lte16_subnet')
lpmpath = getsavepath(fpath, 'lpm_subnet_data')
if (not sndict):
if (os.path.exists(snpath)):
newsndict = blacklist_tools.load_dict(snpath)
newsndict1 = dict(newsndict, **sndict) #merge
saveToJSON(newsndict1, fpath, "remain_subnet")
else:
saveToJSON(sndict, fpath, "remain_subnet")
if (not sn_lte16):
if (os.path.exists(ltepath)):
newlte = blacklist_tools.load_dict(ltepath)
newlte16 = dict(newlte, **sn_lte16) #merge
saveToJSON(newlte16, fpath, 'lte16_subnet')
else:
saveToJSON(sn_lte16, fpath, 'lte16_subnet')
if (not lpmdict):
if (os.path.exists(lpmpath)):
newlpmdict = blacklist_tools.load_dict(lpmpath)
newlpmdict1 = dict(newlpmdict, **lpmdict) #merge
saveToJSON(newlpmdict1, fpath, 'lpm_subnet_data')
else:
saveToJSON(lpmdict, fpath, 'lpm_subnet_data')
#match
subnet_result = []
for ips in es_ip:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
subnet_result.append({ips: 'subnet_lpm_match'})
return subnet_result, sndict, sn_lte16