def load(uname):
import constants, data.data, game.bag
fname = constants.DataPath + uname
out = data.data()
with open(fname) as f:
d = f.readlines()
out.name = d[0]
out.hero = d[1]
out.weapon = int(d[2])
out.point = float(d[3])
out.run_distance = float(d[4])
out.bag = game.bag.bag(None)
out.bag.deserialize(d[5])
print out
return out
def plot_subnets_stats(conf):
"""Visualizes the graphs of subnetworks.
Args:
conf: Dictionary consisting of configuration parameters.
"""
network_type = conf["network"]["type"]
if network_type == "cnn":
network = CNN(conf)
elif network_type == "mlp":
network = MLP(conf)
model = network.build()
_, _, x_test, _ = data(conf)
x_test = x_test[:1000]
# Get directories of all models
dir_names = next(os.walk(conf["paths"]["model"]))[1]
model_dirs = ["{}{}{}".format(conf["paths"]["model"], dir_name, "/model.ckpt")
for dir_name in dir_names if dir_name.split("/")[-1].endswith("1")]
print("{} models found.".format(len(model_dirs)))
# Determine avg and std of subnet activations
subnets_statistics = list()
for model_dir in model_dirs:
model.load_weights(model_dir).expect_partial()
stats = get_preactivation_statistics(model, x_test)
subnets_statistics.append(stats)
# Extract layer names
layer_names = [layer_name for layer_name in subnets_statistics[0].keys()]
print("{} subnets found.".format(len(layer_names)))
stats = {layer_name: {"avg": None, "std": None} for layer_name in layer_names}
# Compute average stats of all subnets
for layer_name in layer_names:
avg = 0.0
std = 0.0
for subnet in subnets_statistics:
avg += subnet[layer_name]["avg"]
std += subnet[layer_name]["std"]
stats[layer_name]["avg"] = avg / len(subnets_statistics)
stats[layer_name]["std"] = std / len(subnets_statistics)
# Extract graphs from subnets from all models
graphs_dict = {layer_name: {"x": list(), "y": list()} for layer_name in layer_names}
for model_dir in model_dirs:
model.load_weights(model_dir).expect_partial()
graphs = get_subnet_graphs(model, stats)
for layer_name in layer_names:
graphs_dict[layer_name]["x"].append(graphs[layer_name]["x"])
graphs_dict[layer_name]["y"].append(graphs[layer_name]["y"])
stats = {layer_name: {"x": None, "y_avg": None, "y_std": None} for layer_name in layer_names}
for layer_name in layer_names:
stats[layer_name]["x"] = np.mean(graphs_dict[layer_name]["x"], axis=0) # not necessary
stats[layer_name]["y_avg"] = np.mean(graphs_dict[layer_name]["y"], axis=0)
stats[layer_name]["y_std"] = np.std(graphs_dict[layer_name]["y"], axis=0)
# Plot results
nrows = 2
ncols = 4
fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(2*ncols, 2*nrows))
for i, (layer_name, ax) in enumerate(zip(layer_names, axes.flatten())):
x = stats[layer_name]["x"]
y_avg = stats[layer_name]["y_avg"]
y_std = stats[layer_name]["y_std"]
x_all = np.squeeze(graphs_dict[layer_name]["x"]).T
y_all = np.squeeze(graphs_dict[layer_name]["y"]).T
ax.plot(x_all, y_all, linewidth=0.3, alpha=0.4, color="green")
ax.plot(x, y_avg, linewidth=1.0, color="green")
ax.fill_between(x, y_avg - y_std, y_avg + y_std, alpha=0.2, color="green")
ax.grid(True, alpha=0.5, linestyle='dotted')
ax.set_title("$s^{{({layer})}}$".format(layer=str(i+1)))
plt.tight_layout()
plt.savefig('{}subnetworks_{}.png'.format(conf["paths"]["results"], "test"), dpi=100)
plt.close(fig)
import numpy as np
np.random.seed(7)
torch.manual_seed(7)
#Choosing mode='paper_exact' to use originally saved CNN as feature extractor
#mode = 'paper_exact'
mode = 'replicate'
#Generate & save features from the selected model
if mode == 'paper_exact':
model = torch.load('./models/cnn_temporal_saved_models_in_paper.pth').to(
'cpu')
else:
model = torch.load('./models/cnn_temporal.pth').to('cpu')
trainloader, validloader = data(is_temporal=True, bs=100, dev='cpu')
N = model.num_layers()
for i, (x, y) in enumerate(trainloader):
print(i, '/', len(trainloader) - 1)
features = model.layer_output(x.unsqueeze(1), N - 4)
torch.save((features.detach().numpy(), y.detach().numpy().reshape(-1, 1)),
'./saved_features/train' + str(i) + '.pth')
for i, (x, y) in enumerate(validloader):
print(i, '/', len(validloader) - 1)
features = model.layer_output(x.unsqueeze(1), N - 4)
torch.save((features.detach().numpy(), y.detach().numpy().reshape(-1, 1)),
'./saved_features/val' + str(i) + '.pth')
#Load saved feature data, uses different local data arrays because
#np.concatenate is slow when concatenated array size is big
def __init__(self):
self.data = data()
self.base = base()
self.url = self.data.kuaidaili
self.bs = None
import mxnet as mx
from data import data
from data.preprocess import preprocess_list
from config import train, verbose
parser = argparse.ArgumentParser(description='')
parser.add_argument('-d', type=str, help='dataset')
parser.add_argument('-o', type=str, default='.', help='output directory')
parser.add_argument('-f', type=str, default='mp4', help='format')
parser.add_argument('-c', type=str, default='', help='country')
args = parser.parse_args()
train.config['dataset'] = args.d
data = data.data(**train.config)
seq, scr = data.next()
seq, scr = preprocess_list(seq, scr)
nds = dict()
nps = dict()
lr = args.LR
device = mx.gpu(args.gpu)
wave = WaveArch()
def __init__(self):
wx.Frame.__init__(self,
parent=None,
title=u'刷搜索排名小工具 v1.0',
size=(920, 640),
style=wx.MINIMIZE_BOX | wx.CLOSE_BOX)
self.data = data()
self.keyworks, self.urlkw, self.proxyType, self.proxyConfig = "", "", "", ""
self.proValue, self.spend = 0, 0
self.note = self.data.note
self.font = wx.Font(12, wx.SWISS, wx.NORMAL, wx.NORMAL, False)
self.update()
self.Bind(wx.EVT_CLOSE, self.OnClickStop)
# 添加drivers到环境变量
if platform.system() == "Darwin":
self.dir = os.environ["HOME"] + os.sep + "drivers" + os.sep
self.gdname = "geckodriver-v0.11.1-macos.tar.gz"
elif platform.system() == "Windows":
self.dir = os.environ["USERPROFILE"] + os.sep + "drivers" + os.sep
self.gdname = "geckodriver-v0.11.1-win64.zip"
os.environ["PATH"] += ':' + self.dir
# 创建定时器
self.beginTime = 0
self.timer = wx.Timer(self)
self.Bind(wx.EVT_TIMER, self.OnTimer, self.timer)
# 运行log
om = wx.StaticBox(self, -1, u"▼ 运行日志:")
self.multiText = wx.TextCtrl(self,
-1,
value=self.note,
size=(500, 490),
style=wx.TE_MULTILINE | wx.TE_READONLY)
self.multiText.SetInsertionPoint(0)
# 功能选择
fcm = wx.StaticBox(self, -1, u"▼ 功能选择:")
self.runTypeBtn = wx.ToggleButton(self,
-1,
label=u'只刷指数',
size=(84, 21))
self.runTypeBtn.SetValue(True)
self.Bind(wx.EVT_TOGGLEBUTTON, self.OnClickRunTypeBtn, self.runTypeBtn)
self.getRankBtn = wx.ToggleButton(self,
-1,
label=u'获取排名',
size=(84, 21))
self.Bind(wx.EVT_TOGGLEBUTTON, self.OnClickGetRankBtn, self.getRankBtn)
self.dndBtn = wx.Button(self, label=u'部署Driver', size=(84, 21))
self.Bind(wx.EVT_BUTTON, self.cpDriver, self.dndBtn)
# 选择搜索引擎: baidu, sm, sogou
sm = wx.StaticBox(self, -1, u"▼ 搜索平台:")
spfList = ["Baidu", "SM", "Sogou"]
self.rb_splatform = wx.RadioBox(self, -1, "", wx.DefaultPosition,
(120, 80), spfList, 3, wx.SL_VERTICAL)
# 选择平台：web，h5
dm = wx.StaticBox(self, -1, u"▼ 运行平台:")
pfList = ["H5", "Web"]
self.rb_platform = wx.RadioBox(self, -1, "", wx.DefaultPosition,
(120, 40), pfList, 2, wx.SL_HORIZONTAL)
# 是否使用模拟浏览器
self.cb_isPhantomjs = wx.CheckBox(self, -1, u"模拟浏览器?",
wx.DefaultPosition, (120, 30))
# 配置目标页面关键词
tm = wx.StaticBox(self, -1, u"▼ 目标页面标题包含关键词:")
self.target_kw = wx.TextCtrl(self, -1, value=u"穷游网", size=(272, 21))
# 选择keywords文件
fm = wx.StaticBox(self, -1, u"▼ 关键词文件路径:")
self.kwText = wx.TextCtrl(self,
-1,
value=u"点击右侧按钮选择文件...",
size=(222, 21))
self.kwText.Disable()
self.kwText.SetFont(self.font)
self.kwBtn = wx.Button(self, label='...', size=(30, 21))
self.Bind(wx.EVT_BUTTON, self.OnOpenKWFile, self.kwBtn)
self.tmpBtn = wx.Button(self, label='+', size=(30, 21))
self.Bind(wx.EVT_BUTTON, self.OnCreateTmpFile, self.tmpBtn)
# 关键词运行次数
rm = wx.StaticBox(self, -1, u"▼ 运行次数:")
self.runTime = wx.CheckBox(self, -1, u"是否统一配置? 输入运行次数:")
self.runText = wx.TextCtrl(self, -1, size=(62, 21))
self.runText.SetEditable(False)
self.runText.SetValue("100")
self.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox_RT, self.runTime)
# 选择代理方式：Local, api，txt
pm = wx.StaticBox(self, -1, u"▼ 代理方式:")
sampleList = ["Local", "API", "TXT"]
self.rb_proxy = wx.RadioBox(self, -1, "", wx.DefaultPosition,
wx.DefaultSize, sampleList, 3)
self.proxyType = self.rb_proxy.GetItemLabel(
self.rb_proxy.GetSelection())
self.Bind(wx.EVT_RADIOBOX, self.EvtRadioBox_Proxy, self.rb_proxy)
self.proxyTextBtn = wx.Button(self, label='...', size=(30, 21))
self.proxyTextBtn.Hide()
self.Bind(wx.EVT_BUTTON, self.OnOpenProxyFile, self.proxyTextBtn)
self.spiderBtn = wx.Button(self, label='+', size=(30, 21))
self.spiderBtn.Hide()
self.Bind(wx.EVT_BUTTON, self.OnSpider, self.spiderBtn)
# 代理DNS，API, TXT配置输入框
self.proxyText = wx.TextCtrl(self,
-1,
value=self.data.proxy_dns,
size=(220, 21))
self.proxyText.SetFont(self.font)
# 代理数量显示
self.apiCount, self.dnsCount = 0, 0
try:
self.dnsCount = self.getProxyCount("Local", self.data.proxy_dns)
except:
self.errInfo(u'Local代理方式下: 并没有获取到代理数量. ', True)
try:
self.apiCount = self.getProxyCount("API", self.data.proxy_api)
except:
self.errInfo(u'API代理方式下: 并没有获取到代理数量. ', True)
self.proxyCount = wx.StaticText(self,
-1,
label=" |%d" % self.dnsCount,
size=(50, 21))
# 版权模块
self.copyRight = wx.StaticText(self, -1, u"©️LiuFei", style=1)
self.spendTime = wx.StaticText(self, -1, u"▶ 耗时: 00:00:00 ")
self.succTime = wx.StaticText(self, -1, u"▶ 成功次数: 0 ")
self.succRatio = wx.StaticText(self, -1, u"▶ 成功率: 0.0 ")
self.proText = wx.StaticText(self, -1, u"▶ 进度:")
self.process = wx.Gauge(self,
-1,
size=(190, 20),
style=wx.GA_HORIZONTAL)
self.Bind(wx.EVT_IDLE, self.Onprocess)
# 运行按钮
self.buttonRun = wx.Button(self, label=u"运行")
self.Bind(wx.EVT_BUTTON, self.OnClickRun, self.buttonRun)
# 终止按钮
self.buttonStop = wx.Button(self, label=u"关闭")
self.Bind(wx.EVT_BUTTON, self.OnClickStop, self.buttonStop)
# 左侧布局
mbox = wx.BoxSizer(wx.VERTICAL)
vbox = wx.BoxSizer(wx.HORIZONTAL)
leftbox = wx.BoxSizer(wx.VERTICAL)
sbmbox = wx.BoxSizer(wx.HORIZONTAL)
funcbox = wx.StaticBoxSizer(fcm, wx.HORIZONTAL)
funcbox.Add(self.runTypeBtn, 0, wx.ALL, 5)
funcbox.Add(self.getRankBtn, 0, wx.ALL, 5)
funcbox.Add(self.dndBtn, 0, wx.ALL, 5)
searchbox = wx.StaticBoxSizer(sm, wx.HORIZONTAL)
searchbox.Add(self.rb_splatform, 0, wx.ALL, 5)
driverbox = wx.StaticBoxSizer(dm, wx.VERTICAL)
driverbox.Add(self.rb_platform, 0, wx.ALL, 5)
driverbox.Add(self.cb_isPhantomjs, 0, wx.ALL, 5)
targetbox = wx.StaticBoxSizer(tm, wx.HORIZONTAL)
targetbox.Add(self.target_kw, 0, wx.ALL, 5)
filebox = wx.StaticBoxSizer(fm, wx.HORIZONTAL)
filebox.Add(self.kwText, 0, wx.ALIGN_LEFT, 5)
filebox.Add(self.kwBtn, 0, wx.ALIGN_RIGHT, 5)
filebox.Add(self.tmpBtn, 0, wx.ALIGN_RIGHT, 5)
runBox = wx.StaticBoxSizer(rm, wx.HORIZONTAL)
runBox.Add(self.runTime, 0, wx.ALL, 5)
runBox.Add(self.runText, 0, wx.ALL, 5)
sbmbox.Add(searchbox, 0, wx.ALIGN_LEFT, 5)
sbmbox.Add(driverbox, 0, wx.ALIGN_RIGHT, 5)
leftbox.Add(funcbox, 0, wx.ALL, 5)
leftbox.Add(sbmbox, 0, wx.ALL, 5)
leftbox.Add(targetbox, 0, wx.ALL, 5)
leftbox.Add(filebox, 0, wx.ALL, 5)
leftbox.Add(runBox, 0, wx.ALL, 5)
proxyBox = wx.StaticBoxSizer(pm, wx.VERTICAL)
proxymodBox = wx.BoxSizer(wx.HORIZONTAL)
proxymodBox.Add(self.rb_proxy, 0, wx.ALL, 5)
proxymodBox.Add(self.proxyTextBtn, 0, wx.CENTER, 5)
proxymodBox.Add(self.spiderBtn, 0, wx.CENTER, 5)
proxyConfBox = wx.BoxSizer(wx.HORIZONTAL)
proxyConfBox.Add(self.proxyText, 0, wx.ALIGN_LEFT, 5)
proxyConfBox.Add(self.proxyCount, 0, wx.ALIGN_RIGHT, 5)
proxyBox.Add(proxymodBox, 0, wx.ALL, 5)
proxyBox.Add(proxyConfBox, 0, wx.ALL, 5)
leftbox.Add(proxyBox, 0, wx.ALL, 5)
# 右侧布局
rightBox = wx.BoxSizer(wx.VERTICAL)
logBox = wx.StaticBoxSizer(om, wx.VERTICAL)
logBox.Add(self.multiText, 0, wx.ALL, 5)
rightBox.Add(logBox, 0, wx.ALL, 5)
# 底部布局
buttomBox = wx.BoxSizer(wx.HORIZONTAL)
processBox = wx.BoxSizer(wx.HORIZONTAL)
btnBox = wx.BoxSizer(wx.HORIZONTAL)
processBox.Add(self.copyRight, 0, wx.ALL, 5)
processBox.Add(self.spendTime, 0, wx.ALL, 5)
processBox.Add(self.succTime, 0, wx.ALL, 5)
processBox.Add(self.succRatio, 0, wx.ALL, 5)
processBox.Add(self.proText, 0, wx.ALL, 5)
processBox.Add(self.process, 0, wx.ALL, 5)
btnBox.Add(self.buttonRun, 0, wx.ALL, 5)
btnBox.Add(self.buttonStop, 0, wx.ALL, 5)
# 整体布局
vbox.Add(leftbox, 0, wx.ALL, 5)
vbox.Add(rightBox, 0, wx.ALL, 5)
buttomBox.Add(processBox, 0, wx.ALL, 5)
buttomBox.Add(btnBox, 0, wx.ALL, 5)
mbox.Add(vbox, 0, wx.ALL, 5)
mbox.Add(buttomBox, 0, wx.ALL, 5)
self.SetSizer(mbox)
mbox.Fit(self)
self.Show()
import warnings
warnings.filterwarnings("ignore")
from tqdm import tqdm
from data.data import data
from model import Model
from sklearn.pipeline import Pipeline
from features.building_features import manual_features
from sklearn.preprocessing import StandardScaler
tqdm.pandas()
if __name__ == '__main__':
"""Loading data"""
loaded_data = data()
loaded_data.load_data()
"""Run model"""
model = Model()
model.run_model(loaded_data)
"""Run Deep text only"""
# from models.CNN import CNN
# deep = CNN()
# deep.prepare_input(loaded_data.train, loaded_data.dev, loaded_data.test)
# deep.Network()
# deep.run_model(reTrain=True)
"""Run Deep text + features"""
# model = Pipeline([
# ('main_pip', Pipeline([
# ('manual_features', manual_features(path='./features', n_jobs=-1)),
# ('Normalization', StandardScaler()),
# ])),
# ])
def __init__(self):
self.data = data()
self.base = base()
self.url = self.data.swei360
self.bs = None
from data.data import data
from core.bot.tools import *
from core.logger import log
from core.bot.time import Time
import traceback
import asyncio
import session
import sys
if __name__ == '__main__':
log.info(f'>{trace.cyan} Starting at {Time.readable.at()}.')
# Initialize database
log.info(f'{trace.cyan}> Initializing {trace.black.s}dataset{trace.cyan} Database.')
try:
data()
log.info(f'{trace.cyan}> Initialized {trace.black.s}dataset{trace.cyan}.')
except Exception as err:
log.warning(f'> Failed to load {trace.black.s}dataset{trace.warn}. Please restart!')
log.error(f'> {short_traceback()}')
log.critical(f'> {traceback.format_exc()}')
# Start API
import api
api.Main()
# Initialize extensions
# Append cCogs
# append_cog('session.py') # Load session
append_cog('debug.py') # Load debug things
append_cog('main.py') # Load essentials