def rename():
filefold = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']
filecount = [131676, 108189, 117665, 159616, 294130]
count = 0
i = 0
while i < len(filefold):
filepath = global_var.get_value(
'filepath') + filefold[i] + '\\dataset\\image\\'
index = 0
while index <= filecount[i]:
filename = filepath + 'flow-' + str(index) + '-image.png'
try:
new_filename = global_var.get_value(
'filepath') + 'dataset\\image\\flow-' + str(
count) + '-image.png'
shutil.copyfile(filename, new_filename)
index += 1
count += 1
except FileNotFoundError:
print("%s not found" % filename)
index += 1
continue
i += 1
def process_one_cat(url, cat_list):
PAGE_NUM_PROCESSING = 1
global_var._init()
global_var.set_value("PAGE_NUM_PROCESSING", PAGE_NUM_PROCESSING)
global_var.set_value("isLastPage", False)
while True:
page_num = global_var.get_value('PAGE_NUM_PROCESSING')
tmp_url = url + str(page_num) + "/"
# getJobList(tmp_url)
print("while True main:" + tmp_url)
s = requests.Session()
lg = Lagou()
pagegen = lg.getJobListPerPage(tmp_url, s)
for item in pagegen:
time_wait = 1 + float(random.randint(1, 100)) / 20
time.sleep(time_wait)
print("休息时间：" + str(time_wait))
for job in item:
db.insert(job, cat_list)
print("跳出生成器")
print("休息5秒钟")
time.sleep(5)
if global_var.get_value("isLastPage"):
print("----------爬取结束---------，共" +
str(global_var.get_value('PAGE_NUM_PROCESSING')) + "页")
break
def __init__(self, images, labels, dtype=dtypes.float32, reshape=True):
"""Construct a DataSet.
`dtype` can be either `uint8` to leave the input as `[0, 255]`,
or `float32` to rescale into `[0, 1]`.
"""
#initilize the image weight and height
image_weight = int(global_var.get_value('image_weight'))
image_height = int(global_var.get_value('image_height'))
dtype = dtypes.as_dtype(dtype).base_dtype
if dtype not in (dtypes.uint8, dtypes.float32):
raise TypeError(
'Invalid image dtype %r, expected uint8 or float32' % dtype)
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
if reshape:
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
image_weight * image_height)
if dtype == dtypes.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def if_catch(rk_i, rk_j, po_i, po_j):
xx = gl.get_value("xx")
yy = gl.get_value("yy")
for i in range(8):
if (rk_i + xx[i] == po_i) and (rk_j + yy[i] == po_j):
return True
return False
def run(self, start, end):
self.open_list.add(end)
while True:
self.process_state()
if start.t == "close":
break
start.set_state("s")
s = start
self.map.draw_map(start)
while s != gl.get_value("end"):
s.set_state("s")
s = s.parent
s.set_state("e")
self.map.print_map()
print("--------------------------")
tmp = start
#self.map.set_obstacle([(2, 1), (2, 2), (2, 3), (2, 4)])
#self.map.erase_obstacle([(2, 0)])
flag = False
while tmp != gl.get_value("end"):
tmp.set_state("*")
rkij = gl.get_value("rkij")
self.map.erase_obstacle(utils.set_obstacle(rkij))
utils.rk_random_move()
rkij = gl.get_value("rkij")
poij = gl.get_value("poij")
#utils.msgbox()
self.map.set_obstacle(utils.set_obstacle(rkij))
nr = gl.get_value("nr")
nc = gl.get_value("nc")
poij = gl.get_value("poij")
lmij = gl.get_value("lmij")
rkij = gl.get_value("rkij")
self.map.print_map()
print("")
if tmp.parent.state == "#":
#self.modify(tmp)
#continue
m = Map(nr, nc)
m.set_obstacle(utils.set_obstacle(rkij))
#start = m.map[0][0]
start = m.map[tmp.x][tmp.y]
end = m.map[lmij[0] - 1][lmij[1] - 1]
###############################################utils.erase_lines()
dstar = Dstar(m)
dstar.run(start, end)
flag = True
return
tmp = tmp.parent
utils.PO_move(tmp.x, tmp.y)
if flag == False:
tmp.set_state("e")
def PO_move(i, j):
aux = index_to_xy([i + 1, j + 1])
cv = gl.get_value("cv")
PO = gl.get_value("PO")
po_cnt = gl.get_value("po_cnt")
po_cnt.set(int(po_cnt.get()) + 1)
cv.coords(PO, (aux[0], aux[1]))
gl.set_value("poij", [i + 1, j + 1])
def save_to_file(**kwargs):
data = kwargs["data"]
path = global_var.get_value("floor_data_path")
file_name = kwargs["file"] + ".json"
full_path = os.path.join(path, file_name)
with open((full_path), "w") as f:
json.dump(data, f)
status_text = global_var.get_value("status_text")
if "show_status" in kwargs:
if kwargs["show_status"] == True:
status_text.set(f"成功保存到{full_path}！")
def index_to_xy(p):
#[1, 1] to [nr, nc]
xi = p[0]
yi = p[1]
ver_int = gl.get_value("ver_int")
top_lef = gl.get_value("top_lef")
lev_int = gl.get_value("lev_int")
lef_lef = gl.get_value("lef_lef")
x = lef_lef + yi + lev_int * (yi - 1)
y = top_lef + xi + ver_int * (xi - 1)
return [x, y]
def rk_follow():
rkij = gl.get_value("rkij")
poij = gl.get_value("poij")
lmij = gl.get_value("lmij")
RK = gl.get_value("RK")
cv = gl.get_value("cv")
add_row = order(lmij[0] - rkij[0])
add_col = order(lmij[1] - rkij[1])
if if_in_range(rkij[0] + add_row, rkij[1] + add_col):
aux = index_to_xy([rkij[0] + add_row, rkij[1] + add_col])
cv.coords(RK, (aux[0], aux[1]))
gl.set_value("rkij", [rkij[0] + add_row, rkij[1] + add_col])
def __init__(self, count=-1):
"""read the pcap file, split the network flow
last version: 2018/3/8
@param count: the packet read, default -1
"""
try:
filename = global_var.get_value("filename")
filepath = global_var.get_value("filepath")
self._pcapreader = PcapReader(filepath + filename)
self._count = count
except Scapy_Exception as e:
print(e)
def msgbox():
rkij = gl.get_value("rkij")
poij = gl.get_value("poij")
lmij = gl.get_value("lmij")
cv = gl.get_value("cv")
if if_catch(rkij[0], rkij[1], poij[0], poij[1]):
messagebox.showwarning("ohno!", "被抓住了，再试一次吧！")
cmd.restart()
elif lmij[0] == poij[0] and lmij[1] == poij[1]:
PO = gl.get_value("PO")
cv.delete(PO)
messagebox.showwarning("ohyeh!", "与人质相遇！向出口前进！")
gl.set_value("if_escape", True)
def loop_role():
sel_role = gl.get_value("sel_role")
if gl.get_value("is_start"):
sel_role += 1
sel_role %= 2
gl.set_value("sel_role", sel_role)
HINT = gl.get_value("HINT")
sel_role_button = gl.get_value("sel_role_button")
if sel_role == 0:
sel_role_button.config(image=gc.get_value("rk_sTK"))
HINT.set("RK移步")
elif sel_role == 1:
sel_role_button.config(image=gc.get_value("police_sTK"))
HINT.set("警察移步")
def music_on_off():
cnt = gl.get_value("music_ctrl")
gl.set_value("music_ctrl", cnt * (-1))
if cnt * (-1) > 0:
filename = 'music/Farm.wav'
pygame.mixer.init()
pygame.mixer.music.load(filename)
pygame.mixer.music.play(loops=-1, start=0.0)
music_button = gl.get_value("music_button")
music_button.config(image=gc.get_value("voiceonTK"))
else:
pygame.mixer.music.stop()
music_button = gl.get_value("music_button")
music_button.config(image=gc.get_value("voiceoffTK"))
def read_floor_index(path=None):
if path == None:
path = global_var.get_value("floor_data_path")
floor_index_path = os.path.join(path, "..", "floor_index.json")
else:
floor_index_path = path.get()
status_text = global_var.get_value("status_text")
try:
with open(floor_index_path) as f:
floor_index = json.load(f)
flush_floor_list(floor_index["index"])
enable_edit()
status_text.set(f"读取{floor_index_path}成功！")
except:
status_text.set(f"读取{floor_index_path}失败！")
def mode0():
nr = gl.get_value("nr")
nc = gl.get_value("nc")
poij = gl.get_value("poij")
lmij = gl.get_value("lmij")
rkij = gl.get_value("rkij")
print(nr, nc)
m = Map(int(nr), int(nc))
m.set_obstacle(utils.set_obstacle(rkij))
start = m.map[poij[0] - 1][poij[1] - 1]
end = m.map[lmij[0] - 1][lmij[1] - 1]
gl.set_value("end", end)
dstar = Dstar(m)
dstar.run(start, end)
utils.msgbox()
def draw_map(self, s):
cv = gl.get_value("cv")
while s != gl.get_value("end"):
sxy = utils.index_to_xy([s.x + 1, s.y + 1])
spxy = utils.index_to_xy([s.parent.x + 1, s.parent.y + 1])
line = cv.create_line(sxy[0],
sxy[1],
spxy[0],
spxy[1],
fill="green",
width=2)
lines.lines.append(line)
#s.set_state("s")
s = s.parent
s.set_state("e")
def init(use_nn=True):
"""初始化语料库等资源"""
product = gl.get_value('PRODUCT', '汽车')
print('正在进行初始化设置...')
ltp_init()
entities, term2entity = load_enititiy(
whole_part_path='./KnowledgeBase/' + product + '/whole-part.txt',
entitiy_synonym_path='./KnowledgeBase/' + product +
'/entity-synonym.txt')
va2attributes, term2attributes, entities = load_attribute(
attribute_description_path='./KnowledgeBase/' + product +
'/attribute-description.txt',
attribute_synonym_path='./KnowledgeBase/' + product +
'/attribute-synonym.txt',
entity_attribute_path='./KnowledgeBase/' + product +
'/entity-attribute.txt',
entities=entities)
print('loading nn model')
# model1 = load_model('libs/aspect-model.h5') if use_nn else None
# model2 = load_model('libs/sentiment-model.h5') if use_nn else None
# nn_kwargs1 = pickle.load(open('libs/aspect-nnargs.pkl', 'rb')) if use_nn else None
# nn_kwargs2 = pickle.load(open('libs/sentiment-nnargs.pkl', 'rb')) if use_nn else None
print('初始化设置成功！\n')
return {
'entities': entities,
'term2entity': term2entity,
'va2attributes': va2attributes,
'term2attributes': term2attributes,
# 'model1': model1,
# 'nn_kwargs1': nn_kwargs1,
# 'model2': model2,
# 'nn_kwargs2': nn_kwargs2
}
def drawUI(self):
self.t.title("加载")
self.t.geometry("600x300")
self.l = tk.Label(self.t,
text="加载地图index",
bg="azure",
font=self.font,
width=15,
height=2)
self.l.pack(side="top", fill=tk.X)
self.l2 = tk.Label(self.t,
text="地图index文件路径",
bg="azure",
font=self.font,
height=2)
self.l2.pack(side="left", expand=True)
self.e1 = tk.Entry(self.t, font=self.font, width=30)
self.path = global_var.get_value("floor_data_path")
self.floor_index_path = os.path.join(self.path, "..",
"floor_index.json")
self.e1.insert(tk.END, f"{self.floor_index_path}")
self.e1.pack(side="left", expand=True)
self.b = tk.Button(self.t,
text="读取地图",
command=partial(read_floor_index, self.e1))
self.b.pack(side="left", expand=True)
def restart():
init.init_var()
po_cnt = gl.get_value("po_cnt")
po_cnt.set("0")
gl.set_value("po_cnt", po_cnt)
lm_cnt = gl.get_value("lm_cnt")
lm_cnt.set("0")
gl.set_value("lm_cnt", lm_cnt)
gl.set_value("if_escape", False)
#rk_cnt = gl.get_value("rk_cnt")
#rk_cnt.set("0")
#gl.set_value("rk_cnt", rk_cnt)
HINT = gl.get_value("HINT")
HINT.set("重置成功！")
redraw()
def get_history_date_time():
# 跨文件全局变量
history_data = global_var.get_value('history_data')
list_date_time = []
# 对历史记录文件进行遍历
for data in history_data:
date_time = data[5]
# 由于Chrome浏览器在sqlite中存储的时间是以1601-01-01 00:00:00 为起始时间点的微妙计数
# 与Unix时间戳存在时间间隔，所以需要转换
unix_time_samp = (date_time / 1000000) - 11644473600
# 中国以北京时间为准，北京时间为UTC+8小时，8小时=28800秒
unix_time_samp += 28800
# 放入list_date_time列表
list_date_time.append(unix_time_samp)
# 将时间转化为标准格式
for i in range(len(list_date_time)):
unix_time_samp = list_date_time[i]
list_date_time[i] = time.strftime("%Y-%m-%d", time.gmtime(unix_time_samp))
# 去重复，set表示集合，集合不可能存在重复元素
list_unique = list(set(list_date_time))
# 升序排序
list_unique_sort = sorted(list_unique)
# print(list_unique_sort)
return list_unique_sort
def load_rolelable():
root = gl.get_value("root")
po_cnt = tk.StringVar()
po_cnt.set("0")
gl.set_value("po_cnt", po_cnt)
police_lable_img = tk.Label(root,
image=gc.get_value("policeTK"),
width=70,
height=70)
police_lable = tk.Label(root,
image=gc.get_value("boardTK"),
font=("Helvetica", 34, "bold"),
textvariable=po_cnt,
compound=tk.CENTER,
height=60)
police_lable_img.place(relx=0.7, rely=0.07)
police_lable.place(relx=0.8, rely=0.07)
lm_cnt = tk.StringVar()
lm_cnt.set("0")
gl.set_value("lm_cnt", lm_cnt)
lm_lable_img = tk.Label(root,
image=gc.get_value("lmTK"),
width=50,
height=50)
lm_lable = tk.Label(root,
image=gc.get_value("boardTK"),
textvariable=lm_cnt,
font=("Helvetica", 34, "bold"),
compound=tk.CENTER,
height=60)
lm_lable_img.place(relx=0.713, rely=0.21)
lm_lable.place(relx=0.8, rely=0.2)
def load_start_button():
root = gl.get_value("root")
start_button = tk.Button(root,
image=gc.get_value("startTK"),
relief=tk.FLAT,
bd=0,
command=cmd.start)
start_button.place(relx=0.72, rely=0.37)
def load_setting_button():
root = gl.get_value("root")
setting_button = tk.Button(root,
image=gc.get_value("settingTK"),
relief=tk.FLAT,
bd=0,
command=cmd.setting)
setting_button.place(relx=0.94, rely=0.5)
def load_jintao():
root = gl.get_value("root")
tao_lable = tk.Label(
root,
bd=0,
font=("Helvetica", 9, "bold"),
text="Build by Jintao_Qu, 2018",
).place(relx=0.025, rely=0.965)
def sel_role():
sel_role = gl.get_value("sel_role")
sel_role = (sel_role + 1) % 3
gl.set_value("sel_role", sel_role)
HINT = gl.get_value("HINT")
sel_role_button = gl.get_value("sel_role_button")
if (gl.get_value("if_start") == False):
if sel_role == 0:
sel_role_button.config(image=gc.get_value("rk_sTK"))
HINT.set("设置RK的位置")
elif sel_role == 1:
sel_role_button.config(image=gc.get_value("police_sTK"))
HINT.set("设置警察的位置")
elif sel_role == 2:
sel_role_button.config(image=gc.get_value("lm_sTK"))
HINT.set("设置小拉姆的位置")
def init_role_position():
cv = gl.get_value("cv")
nr = int(gl.get_value("nr"))
nc = int(gl.get_value("nc"))
#init rk's position
rk_i = random.randint(1, nr)
rk_j = random.randint(1, nc)
while (rk_i == nr and rk_j == nc):
rk_i = random.randint(1, nr)
rk_j = random.randint(1, nc)
rkxy = index_to_xy([rk_i, rk_j])
gl.set_value("rkij", [rk_i, rk_j])
#print("RK position", [rk_i, rk_j])
#init police's position
po_i = random.randint(1, nr)
po_j = random.randint(1, nc)
while (if_catch(rk_i, rk_j, po_i, po_j) or (rk_i == po_i and rk_j == po_j)
or (po_i == nc and po_j == nr)):
po_i = random.randint(1, nr)
po_j = random.randint(1, nc)
poxy = index_to_xy([po_i, po_j])
gl.set_value("poij", [po_i, po_j])
#print("PO position", [po_i, po_j])
# init lm's position
lm_i = random.randint(1, nr)
lm_j = random.randint(1, nc)
while (lm_i == po_i and lm_j == po_j) or (
lm_i == rk_i and lm_j == rk_j) or (lm_i == nr and lm_j == nc):
lm_i = random.randint(1, nr)
lm_j = random.randint(1, nc)
lmxy = index_to_xy([lm_i, lm_j])
gl.set_value("lmij", [lm_i, lm_j])
#print("LM position", [lm_i, lm_j])
RK = cv.create_image(rkxy[0], rkxy[1], image=gc.get_value("rk_sTK"))
PO = cv.create_image(poxy[0], poxy[1], image=gc.get_value("police_sTK"))
LM = cv.create_image(lmxy[0], lmxy[1], image=gc.get_value("lm_sTK"))
EX = cv.create_image(565, 465, image=gc.get_value("exitTK"))
gl.set_value("RK", RK)
gl.set_value("PO", PO)
gl.set_value("LM", LM)
gl.set_value("EX", EX)
def load_restart_button():
root = gl.get_value("root")
restart_button = tk.Button(root,
image=gc.get_value("restartTK"),
relief=tk.FLAT,
bd=0,
command=cmd.restart)
gl.set_value("restart_button", restart_button)
restart_button.place(relx=0.92, rely=0.44)
def start_crawl():
while True:
try:
uapools = [
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"
]
headers = {
'Connection': 'Keep-Alive',
'Accept': 'text/html,*/*',
'User-Agent': random.choice(uapools)
}
# 发送第一次请求，发送搜索关键字
opener = crawler.first_request("爬虫", headers)
# 发送第二次请求，返回第一页搜索结果列表页
first_page_response = crawler.second_request(opener, headers)
# 如果有上次运行的记录process_record.log，从记录页码19开始运行
print "上次在第", global_var.get_value("PAGE_NUM_PROCESSING"), "页失败"
opener.addheaders = [("Referer", first_page_response.url)]
page_n_response = opener.open(
"http://kns.cnki.net/kns/brief/brief.aspx?curpage=" +
str(global_var.get_value("PAGE_NUM_PROCESSING")) +
"&RecordsPerPage=20&QueryID=0&ID=&turnpage=1&tpagemode=L&dbPrefix=SCDB&Fields=&DisplayMode=listmode&PageName=ASP.brief_default_result_aspx"
)
page_n_html = page_n_response.read()
page_n_treedata = etree.HTML(page_n_html)
opener.addheaders = [("Referer", page_n_response.url)]
localdir = "C:/code/tmp/pdf/"
crawler.start_download_from_n(
page_n_treedata, opener, localdir,
global_var.get_value("PAGE_NUM_PROCESSING"))
except Exception as error:
print error
print "爬取到第", str(global_var.get_value("PAGE_NUM_PROCESSING")), "失败，重新等待5分钟后，重新从第",\
str(global_var.get_value("PAGE_NUM_PROCESSING")), "页开始爬取数据"
time.sleep(300)
def drawUI(self):
self.t.title("编辑楼层信息")
self.t.geometry("500x300")
self.floor_listbox = global_var.get_value("floor_listbox")
self.floor_data_path = global_var.get_value("floor_data_path")
# 获取当前选中的楼层
self.floor_index = self.floor_listbox.get(
self.floor_listbox.curselection())
try:
self.full_path = os.path.join(self.floor_data_path,
self.floor_index + ".json")
with open(self.full_path) as f:
self.floor_data = json.load(f)
except:
self.status_text.set(
f"读取{self.floor_index}失败，请检查文件是否存在于/project/floors！")
self.data = self.floor_data
self.draw_editor()
def trans_img():
"""pre-process the pcap file and transfer to the image and label file"""
#split flow
pcapreader = pcap_reader.pcap_reader()
pcapreader.flow_split()
for i in range(pcapreader.get_count()):
pcapreader.flow_statistic(i)
del pcapreader
#traverse the file in the file root ".\dataset\image":
root = global_var.get_value('filepath') + "flow"
i = 0
for fn in os.listdir(root):
image = pcap_reader.pcap2img(i, int(global_var.get_value('delta')))
i += 1
#transfer the file to the label
label = pcap_reader.netflow2label()
label.write_file()
