def train(cfg):
logger = setup_logger(name='Train', level=cfg.LOGGER.LEVEL)
logger.info(cfg)
model = build_model(cfg)
device = torch.device(cfg.MODEL.DEVICE)
model.to(device)
#model.to(cuda_device = 'cuda:9')
criterion = build_loss(cfg)
optimizer = build_optimizer(cfg, model)
scheduler = build_lr_scheduler(cfg, optimizer)
train_loader = build_data(cfg, is_train=True)
val_loader = build_data(cfg, is_train=False)
logger.info(train_loader.dataset)
logger.info(val_loader.dataset)
arguments = dict()
arguments["iteration"] = 0
checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD
checkpointer = Checkpointer(model, optimizer, scheduler, cfg.SAVE_DIR)
do_train(cfg, model, train_loader, val_loader, optimizer, scheduler,
criterion, checkpointer, device, checkpoint_period, arguments,
logger)
def exec_model(config, vocab_enc, vocab_dec, vocab_dec_bw, model, line):
enc_inputs = data.build_data(vocab_enc, [line], config.n_enc_seq)
dec_inputs = data.build_data(vocab_enc, [""], config.n_dec_seq)
labels = dec_inputs[:, 1:]
enc_inputs = enc_inputs[:, :-1]
dec_inputs = dec_inputs[:, :-1]
enc_inputs = torch.tensor(enc_inputs, dtype=torch.long).to(config.device)
dec_inputs = torch.tensor(dec_inputs, dtype=torch.long).to(config.device)
for i in range(config.n_dec_seq):
dec_logits, _, _, _ = model(enc_inputs, dec_inputs)
_, index = dec_logits.max(dim=2)
dec_inputs[0][i + 1] = index[0][i]
if index[0][i] == vocab_dec["<eos>"]:
break
output = []
for line in dec_inputs:
out_line = []
for i in line:
i = i.item()
if i == vocab_dec["<bos>"]:
pass
elif i == vocab_dec["<eos>"]:
break
else:
out_line.append(vocab_de_bw[i])
output.append(out_line)
print(output)
def withdraw_view():
param = build_data()
if request.method == "POST":
success = False
message = "Unknown"
try:
amount = int(request.form["amount"])
user_message = request.form["message"]
if my.check_bank(amount):
append_log("Transaction queued ({})".format(amount))
sleep(SLEEP_TIME)
my.set_money(my.get_money() + amount)
my.set_bank(my.get_bank() - amount)
message = "Transaction successful ({}{})".format(
amount, " / {}".format(user_message)
if len(user_message) > 0 else "")
success = True
else:
message = "Not enough money"
except:
message = "Cannot decode amount"
append_log("Withdrawal {} -> {}".format(
"Failed" if not success else "Success", message))
param["message"] = message
param["success"] = success
# return redirect(url_for("log_view"))
return render_template("withdraw.html", **param)
def __init__(self, path):
self.label, self.edge, self.w = build_data(path)
self.V = len(self.label)
self.d = 0.85
self.in_link = [0] * self.V
self.out_link = self.in_link
for i in xrange(self.V):
self.out_link[i] = len(self.edge[i])
for v in self.edge[i]:
self.in_link[v] += 1
# coding:utf-8
import numpy as np
from sklearn import svm
import data
from sklearn.externals import joblib
from sklearn.metrics import classification_report
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
X_train,Y_train = data.build_data('/home/liu/新论文/语料/整理中/初步完成/用于评价',embedding_size=30,max_len=25)
# X_test,Y_test = data.build_dev('/Users/liutong/新论文/语料/整理中/初步完成/test_set',embedding_size=30,max_len=25)
def f1(p,r):
return 2*p*r/(p+r)
# 通过调参数改变拟合的情况
#--------------------------------------SVM-----------------------------------------
clf = svm.SVC(C=31)
clf.fit(X_train,Y_train)
joblib.dump(clf, "svm_model.m") # 保存模型
y_pred_svm = clf.predict(X_train)
# print (len(y_pred))
# print(classification_report(y_pred_svm,Y_test))
p_svm = precision_score(Y_train,y_pred_svm,average='macro')
r_svm = recall_score(Y_train,y_pred_svm,average='macro')
print('SVM-P: ',p_svm)
print('SVM-R: ',r_svm)
def main_view():
return render_template("index.html", **build_data())
def log_view():
return render_template("log.html", **build_data())
def wallet_view():
return render_template("wallet.html", **build_data())
def _generate_data(file, pos_file, dim, window, threads, min_count):
"""Run word2vec in python
Train word2vec by google c-style word2vec.
Vector file name is like 'vector_($dim)_($window)_($mini_count).txt'.
Vocab file name is like 'vocab_($dim)_($window)_($mini_count).txt'.
Both of two will be located at ../bin/
Args:
file: string, input raw corpus
dim: int, dimension of the embedding
window: int, half size of word window
threads: int, threads to run word2vec
mini_count: minimum count of words
Return:
flag: string, the flag specify the embedding feature.The flag format
as _$(dim)_$(window)_$(mini_count)
"""
data_dir = "../data"
input_dir = "%s/input" % data_dir
temp_dir = "%s/temp" % data_dir
def my_signal_handler(signal, frame):
if signal == Signal.SIGINT:
os.system('rm %s/*%s*')
if not os.path.exists(data_dir):
os.mkdir(data_dir)
if not os.path.exists(input_dir):
os.mkdir(input_dir)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
flag = '_%d_%d_%d' % (dim, window, min_count)
save_vector = flag.join(['%s/vector' % input_dir, '.txt'])
save_vocab = flag.join(['%s/vocab' % input_dir,'.txt'])
save_vector_bin = flag.join(['%s/vector' % input_dir,'.bin'])
if not os.path.exists(save_vector) and \
not os.path.exists(save_vector_bin):
command = ("../bin/word2vec -train %s -output %s -cbow 0 -size "
"%d -window %d"
" -hs 1 -negative 0 -threads %d -min-count "
"%d -save-vocab %s") % \
(file, save_vector, dim, window, threads, min_count,
save_vocab)
os.system(command)
#convert a float list to numpy array
def _list_to_ndarray(l):
L = len(l)
vec = np.empty(shape=L, dtype=np.float64)
for i, e in enumerate(l):
vec[i] = np.float64(e)
return vec
if not os.path.exists(save_vector_bin):
with open(save_vector, 'r') as f, \
open(save_vector_bin, 'wb') as g:
title = f.readline()
n = int(title.strip().split()[0])
sys.stdout.write("\n...transform data embedding format\n")
for i, line in enumerate(f):
_print_bar(i+1,n)
line = line.strip().split()[1:]
vec = _list_to_ndarray(line)
g.write(vec.tostring())
sys.stdout.flush()
os.system(" ".join(["rm", save_vector]))
#generate train and test set
if not os.path.exists("%s/train_y%s.txt" % (temp_dir, flag)):
build_data(pos_file, save_vocab, save_vector_bin, flag, dim)
return flag, input_dir