def evaluation(args):
source = pickle_load(os.path.join(args.model_path, 'source.pkl'))
target = pickle_load(os.path.join(args.model_path, 'target.pkl'))
target_test = pickle_load(os.path.join(args.model_path, 'target_test.pkl'))
setting = load_setting(os.path.join(args.model_path, 'setting.yaml'))
start_id, end_id = setting['start_id'], setting['end_id']
type_size = setting['type_size']
player_size = setting['player_size']
team_size = setting['team_size']
detail_size = setting['detail_size']
detail_dim = setting['detail_dim']
src_embed = setting['src_embed']
event_size = setting['event_size']
vocab_size = setting['vocab_size']
trg_embed = setting['trg_embed']
hidden = setting['hidden']
start_id = setting['start_id']
end_id = setting['end_id']
class_weight = None
mlp_layers = setting['mlp_layers']
max_length = setting['max_length']
dropout = setting['dropout']
loss_weight = None
disc_loss = setting['disc_loss']
loss_func = setting['loss_func']
net = setting['net']
dataset = setting['dataset']
numbering = setting['numbering']
reverse_decode = setting['reverse_decode']
home_player_tag = target.word_to_id.get(target.home_player_tag)
away_player_tag = target.word_to_id.get(target.away_player_tag)
home_team_tag = target.word_to_id.get(target.home_team_tag)
away_team_tag = target.word_to_id.get(target.away_team_tag)
test = OptaDataset(path=dataset + '.test',
fields={
'source': source,
'target': target_test
})
test20 = OptaDataset(path=dataset + '.test',
fields={
'source': source,
'target': target_test
},
limit_length=20)
test15 = OptaDataset(path=dataset + '.test',
fields={
'source': source,
'target': target_test
},
limit_length=15)
test10 = OptaDataset(path=dataset + '.test',
fields={
'source': source,
'target': target_test
},
limit_length=10)
if 'disc' in net:
content_word_size = len(target.content_word_to_id)
print('vocab size: {}'.format(vocab_size))
if net == 'plain':
model = MLPEncoder2AttentionDecoder(type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
reverse_decode=reverse_decode)
elif net == 'tmpl':
model = MLPEncoder2AttentionDecoder(type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
source.id_to_player,
home_player_tag,
away_player_tag,
source.id_to_team,
home_team_tag,
away_team_tag,
target.player_to_id,
target.players,
reverse_decode=reverse_decode)
elif net == 'gate':
model = MLPEncoder2GatedAttentionDecoder(type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
reverse_decode=reverse_decode)
elif net == 'gate-tmpl':
model = MLPEncoder2GatedAttentionDecoder(type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
source.id_to_player,
home_player_tag,
away_player_tag,
source.id_to_team,
home_team_tag,
away_team_tag,
target.player_to_id,
target.players,
reverse_decode=reverse_decode)
elif net == 'disc':
model = DiscriminativeMLPEncoder2AttentionDecoder(
type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
content_word_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
loss_weight,
disc_loss,
loss_func,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
reverse_decode=reverse_decode)
elif net == 'disc-tmpl':
model = DiscriminativeMLPEncoder2AttentionDecoder(
type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
content_word_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
loss_weight,
disc_loss,
loss_func,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
source.id_to_player,
home_player_tag,
away_player_tag,
source.id_to_team,
home_team_tag,
away_team_tag,
target.player_to_id,
target.players,
reverse_decode=reverse_decode)
elif net == 'gate-disc':
model = DiscriminativeMLPEncoder2GatedAttentionDecoder(
type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
content_word_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
loss_weight,
disc_loss,
loss_func,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
reverse_decode=reverse_decode)
elif net == 'gate-disc-tmpl':
model = DiscriminativeMLPEncoder2GatedAttentionDecoder(
type_size,
player_size,
team_size,
detail_size,
detail_dim,
src_embed,
event_size,
vocab_size,
content_word_size,
trg_embed,
hidden,
start_id,
end_id,
class_weight,
loss_weight,
disc_loss,
loss_func,
mlp_layers,
max_length,
dropout,
IGNORE_LABEL,
source.id_to_player,
home_player_tag,
away_player_tag,
source.id_to_team,
home_team_tag,
away_team_tag,
target.player_to_id,
target.players,
reverse_decode=reverse_decode)
if numbering:
model.player_id = target.player_id
model.team_id = target.team_id
# load best model
if args.gpu is not None:
model.use_gpu(args.gpu)
model.id_to_word = target.id_to_word
model.load_model(os.path.join(args.model_path, 'best.model'))
batch_size = args.batch
src_test_iter = SequentialIterator(test.source,
batch_size,
None,
event_size,
source.fillvalue,
gpu=args.gpu)
src_test20_iter = SequentialIterator(test20.source,
batch_size,
None,
event_size,
source.fillvalue,
gpu=args.gpu)
src_test15_iter = SequentialIterator(test15.source,
batch_size,
None,
event_size,
source.fillvalue,
gpu=args.gpu)
src_test10_iter = SequentialIterator(test10.source,
batch_size,
None,
event_size,
source.fillvalue,
gpu=args.gpu)
trg_test_iter = Iterator(test.target,
batch_size,
wrapper=EndTokenIdRemoval(end_id),
gpu=None)
trg_test20_iter = Iterator(test20.target,
batch_size,
wrapper=EndTokenIdRemoval(end_id),
gpu=None)
trg_test15_iter = Iterator(test15.target,
batch_size,
wrapper=EndTokenIdRemoval(end_id),
gpu=None)
trg_test10_iter = Iterator(test10.target,
batch_size,
wrapper=EndTokenIdRemoval(end_id),
gpu=None)
with open('./dataset/player_list.json.new') as f:
id_to_player = json.load(f)
with open('./dataset/team_list.json.new') as f:
id_to_team = json.load(f)
def convert(ind, no_tag=False):
if 'player' in ind:
if no_tag:
i = ind.replace('player', '')
return id_to_player.get(i, ind)
else:
return ind
elif 'team' in ind:
if no_tag:
i = ind.replace('team', '')
return id_to_team.get(i, ind)
else:
return ind
else:
return ind
if 'disc' in net:
bleu_score, accuracy, hypotheses = evaluate_bleu_and_accuracy(
model, src_test_iter, trg_test_iter)
bleu_score20, _, hypotheses20 = evaluate_bleu_and_accuracy(
model, src_test20_iter, trg_test20_iter)
bleu_score15, _, hypotheses15 = evaluate_bleu_and_accuracy(
model, src_test15_iter, trg_test15_iter)
bleu_score10, _, hypotheses10 = evaluate_bleu_and_accuracy(
model, src_test10_iter, trg_test10_iter)
else:
bleu_score, hypotheses = evaluate_bleu(model, src_test_iter,
trg_test_iter)
bleu_score20, hypotheses20 = evaluate_bleu(model, src_test20_iter,
trg_test20_iter)
bleu_score15, hypotheses15 = evaluate_bleu(model, src_test15_iter,
trg_test15_iter)
bleu_score10, hypotheses10 = evaluate_bleu(model, src_test10_iter,
trg_test10_iter)
print('best score: {}'.format(bleu_score))
print('best score20: {}'.format(bleu_score20))
print('best score15: {}'.format(bleu_score15))
print('best score10: {}'.format(bleu_score10))
# save hypothesis
hypotheses_for_save = [
' '.join([convert(y, True) for y in h]) for h in hypotheses
]
hypotheses20_for_save = [
' '.join([convert(y, True) for y in h]) for h in hypotheses20
]
hypotheses15_for_save = [
' '.join([convert(y, True) for y in h]) for h in hypotheses15
]
hypotheses10_for_save = [
' '.join([convert(y, True) for y in h]) for h in hypotheses10
]
references_for_save = [
' '.join(convert(y, True) for y in r[0]) for r in test.target
]
references20_for_save = [
' '.join(convert(y, True) for y in r[0]) for r in test20.target
]
references15_for_save = [
' '.join(convert(y, True) for y in r[0]) for r in test15.target
]
references10_for_save = [
' '.join(convert(y, True) for y in r[0]) for r in test10.target
]
TextFile(os.path.join(args.model_path, 'hypo'), hypotheses_for_save).save()
TextFile(os.path.join(args.model_path, 'hypo_len20'),
hypotheses20_for_save).save()
TextFile(os.path.join(args.model_path, 'hypo_len15'),
hypotheses15_for_save).save()
TextFile(os.path.join(args.model_path, 'hypo_len10'),
hypotheses10_for_save).save()
TextFile(os.path.join('./dataset', 'ref'), references_for_save).save()
TextFile(os.path.join('./dataset', 'ref_len20'),
references20_for_save).save()
TextFile(os.path.join('./dataset', 'ref_len15'),
references15_for_save).save()
TextFile(os.path.join('./dataset', 'ref_len10'),
references10_for_save).save()
# generate readable text
result = []
for ref, hyp in zip(test.target.data, hypotheses):
if type(ref) == tuple:
ref = ref[0]
ref = ' '.join([convert(y) for y in ref]).split()
try:
bleu_score = sentence_bleu(
[ref], hyp, smoothing_function=SmoothingFunction().method1)
except:
bleu_score = 0
ref = ' '.join([convert(y, True) for y in ref]).split()
hyp = ' '.join([convert(y, True) for y in hyp]).split()
result.append((' '.join(ref), ' '.join(hyp), bleu_score))
inputs = []
for xs in test20.source.data:
data = []
for x in xs[:5]:
event = event_type_mapper.get(x[0], x[0])
player = id_to_player.get(str(x[1]), x[1])
team = id_to_team.get(str(x[2]), x[2])
detail = ','.join(
[qualifier_type_mapper.get(i[-1], i[-1]) for i in x[-1]])
data.append('event: {} player: {} team: {} detail: {}'.format(
event, player, team, detail))
inputs.append('\n'.join(data))
result = [[x, *y] for x, y in zip(inputs, result)]
result = sorted(result, key=lambda x: -x[-1])
TextFile(os.path.join(args.model_path, 'test20_gate_disc_tmpl.txt'), [
'src:\n{}\nref: {}\nhyp: {}\nbleu: {}\n##\n'.format(*x) for x in result
]).save()
def main():
parser = ArgumentParser(description='train a seq2seq model',
formatter_class=RawTextHelpFormatter)
parser.add_argument('CONFIG',
default=None,
type=str,
help='path to config file')
parser.add_argument('--gpu',
'-g',
default=None,
type=str,
help='gpu numbers\nto specify')
parser.add_argument('--debug',
default=False,
action='store_true',
help='switch to debug mode')
args = parser.parse_args()
with open(args.CONFIG, "r") as f:
config = json.load(f)
os.makedirs(os.path.dirname(config['arguments']['save_path']),
exist_ok=True)
source_id_to_word, target_id_to_word, model, device, train_data_loader, valid_data_loader, optimizer = \
load_setting(config, args)
n_pred = 5
n_sample = 1 if model == Seq2seq else 5
threshold = 5
clip = 1e-2
bar = ProgressBar(0, len(train_data_loader))
for epoch in range(1, config['arguments']['epoch'] + 1):
print(f'*** epoch {epoch} ***')
# train
model.train()
annealing = sigmoid(epoch - threshold)
total_loss = 0
total_rec_loss = 0
total_reg_loss = 0
total_c_loss = 0
for batch_idx, (source, source_mask, target_inputs, target_outputs, target_mask) \
in enumerate(train_data_loader):
bar.update(batch_idx)
source = source.to(device)
source_mask = source_mask.to(device)
target = target_inputs.to(device)
target_mask = target_mask.to(device)
label = target_outputs.to(device)
# Forward pass
loss, details = model(source, source_mask, target, target_mask,
label, annealing)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
optimizer.step()
total_loss += loss
total_rec_loss += details[0]
total_reg_loss += details[1]
total_c_loss += details[2]
else:
print('')
print(f'train_loss={total_loss / (batch_idx + 1):.3f}'
f'/rec:{total_rec_loss / (batch_idx + 1):.3f}'
f'/reg:{total_reg_loss / (batch_idx + 1):.3f}'
f'/c:{total_c_loss / (batch_idx + 1):.3f}')
# validation
model.eval()
with torch.no_grad():
total_loss = 0
total_rec_loss = 0
total_reg_loss = 0
total_c_loss = 0
for batch_idx, (source, source_mask, target_inputs, target_outputs, target_mask) \
in enumerate(valid_data_loader):
source = source.to(device)
source_mask = source_mask.to(device)
target = target_inputs.to(device)
target_mask = target_mask.to(device)
label = target_outputs.to(device)
loss, details = model(source, source_mask, target, target_mask,
label, annealing)
total_loss += loss
total_rec_loss += details[0]
total_reg_loss += details[1]
total_c_loss += details[2]
else:
print(f'valid_loss={total_loss / (batch_idx + 1):.3f}'
f'/rec:{total_rec_loss / (batch_idx + 1):.3f}'
f'/reg:{total_reg_loss / (batch_idx + 1):.3f}'
f'/c:{total_c_loss / (batch_idx + 1):.3f}')
random_indices = choice(np.arange(len(source)),
n_pred,
replace=False)
print(random_indices)
s_translation = translate(source[random_indices],
source_id_to_word,
is_target=False)
t_translation = translate(target[random_indices],
target_id_to_word,
is_target=True)
p_translation = \
[translate(model.predict(source, source_mask)[random_indices], target_id_to_word,is_target=True)
for _ in range(n_sample)]
p_translation = list(zip(*p_translation))
for s, t, ps in zip(s_translation, t_translation,
p_translation):
print(f'source:{" ".join(s)} / target:{" ".join(t)}')
for i, p in enumerate(ps):
print(f'predict{i+1}:{" ".join(p)}')
# TODO: add metrics
torch.save(model.state_dict(),
os.path.join(config['arguments']['save_path'], f'sample.pth'))
# !/usr/bin/env python # @Time : 2019-08-29 17:50 # @Author : [email protected] # @Site : # @File : common.py import asyncio from sanic import Sanic from jinja2 import Environment, FileSystemLoader, select_autoescape from utils import load_setting from connections import RedisConnectionPool __all__ = ["app"] app = Sanic(__name__) app.config = load_setting() app.template = Environment(loader=FileSystemLoader( app.config["TEMPLATES_PATH"]), autoescape=select_autoescape(['html', 'xml']), enable_async=False) app.static("/static", "./static") @app.listener('before_server_start') async def before_server_start(app, loop): queue = asyncio.Queue() app.queue = queue app.redis = await RedisConnectionPool(loop=loop ).init(app.config['REDIS_CONFIG'])
from sqlalchemy import create_engine, INT, VARCHAR, Column, DATETIME, FLOAT
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from utils import load_setting, json
setting = load_setting()
def new_db(name):
if not setting['db_status']:
url = setting['url']
engine = create_engine(url) # ,echo =True)
conn = engine.connect()
conn.execute("commit")
try:
conn.execute(f"CREATE DATABASE {name}")
except Exception as e:
print(e)
conn.close()
setting['db_status'] = 1
with open('setting.json', 'w') as f:
json.dump(setting, f)
# 建立数据库后注释
new_db('jd_price')
# 初始化用
url = setting['url'] + 'jd_price'
engine = create_engine(url) #,echo =True)
Base = declarative_base(engine)
#创建会话
def Cloud189SignGUI():
utils.load_setting()
sg.theme(config.DEFAULT_THEME)
layout = [[sg.Text('Cloud189 Signer', size=(40, 1), font=('Any 15'))],
[
sg.T('账号,多个账号用#号分割:', font=config.default_font),
sg.Input(default_text=config.username,
size=(40, 1),
key='username')
],
[
sg.T('密码,多个密码用#号分割:', font=config.default_font),
sg.Input(default_text=config.password,
size=(40, 1),
key='password')
], [sg.Button('签到', key='Sign')],
[sg.Output(size=(90, 20), font=config.default_font)],
[
sg.Button('退出',
key='Exit',
button_color=('white', 'firebrick3')),
sg.Button('源码',
key='home_page',
button_color=('white', 'springgreen4'))
]]
window = sg.Window('天翼云签到',
layout,
text_justification='r',
default_element_size=(15, 1),
font=('Any 14'))
while True:
event, values = window.read()
if event in ('Exit', None):
utils.save_setting()
break # exit button clicked
if event == 'Sign':
window.refresh()
username = values['username']
password = values['password']
err = False
if username == "":
err = True
print("账号不能为空!")
if password == "":
err = True
print("密码不能为空!")
if not err:
user_list = username.split("#")
pass_list = password.split("#")
if len(user_list) != len(pass_list):
print("账号和密码个数不对应!")
else:
config.username = username
config.password = password
print('开始签到....')
try:
for i in range(len(user_list)):
str_paras = utils.to_command_paras(
user_list[i], pass_list[i])
thread_download = Thread(
target=checkin.main,
args=[shlex.split(str_paras)])
thread_download.start()
except Exception as e:
print(e)
elif event == 'home_page':
webbrowser.open_new('https://github.com/xiaogouxo/cloud189-signer')
window.close()