def get_metrics(self, scores, labels, prefix):
scores = np.asarray(scores)
labels = np.asarray(labels)
metrics = {'{}_logloss'.format(prefix): log_loss(y_true=labels, y_pred=scores),
'{}_auc'.format(prefix): roc_auc_score(y_true=labels, y_score=scores)}
pred_labels = (scores > 0.5).astype(int)
metrics['{}_accuracy'.format(prefix)] = np.sum(pred_labels == labels) / len(labels)
return metrics
def get_metrics(self, score, label, perfix):
score = np.asarray(score)
label = np.asarray(label)
metrics = {
'{}_logloss'.format(perfix): log_loss(label, score),
'{}_auc'.format(perfix): roc_auc_score(label, score)
}
pred_labels = (score > 0.5).astype(int)
metrics['{}_accuracy'.format(perfix)] = np.sum(
pred_labels == label) / len(label)
return metrics
def anomaly_detection(anomaly_score_dict, name, df):
X=[]
Y=[]
for l in anomaly_score_dict:
X += anomaly_score_dict[l]
if l in inside_labels:
y += [0]*len(anomaly_score_dict[l])
else:
y += [1]*len(anomaly_score_dict[l])
x = np.array(X)
y = np.array(Y)
X, y = utils.shuffle(X, y, random_state=0)
X_train = X[:len(X)/2]
y_train = y[:len(y)/2]
clf = linear_model.LogisticRegressor(C=1.0)
clf.fit(X_train, y_train)
auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proda(np.array(X_test))[:,1])
print("AUC", auc)
df.set_value()
def generate_classification_perf(truths, pred_probs, multiclass=False):
"""Given truths, and predicted probabilities, generate ModelPerf object"""
pred_classes = np.round(pred_probs).astype(int)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
retval = ClassificationModelPerf(
auroc=metrics.roc_auc_score(truths, pred_probs),
auroc_curve=metrics.roc_curve(truths, pred_probs)
if not multiclass else None,
auprc=metrics.average_precision_score(truths, pred_probs),
accuracy=metrics.accuracy_score(truths, pred_classes)
if not multiclass else None,
recall=metrics.recall_score(truths, pred_classes)
if not multiclass else None,
precision=metrics.precision_score(truths, pred_classes)
if not multiclass else None,
f1=metrics.f1_score(truths, pred_classes)
if not multiclass else None,
ce_loss=metrics.log_loss(truths, pred_probs, normalize=False) /
np.prod(truths.shape),
)
return retval
print(X_test.shape)
print("Y_test.shape: ")
print(Y_test.shape)
In [30]:
# 3.1 建立逻辑回归模型，并且设定参数
lr_model= LogisticRegression(penalty='l2', C=1000, solver='lbfgs', max_iter=500)
# 3.2 训练逻辑回归模型
lr_model.fit(X_train,Y_train.values.ravel())
In [31]:
# 3.3 采用测试集验证模型离线指标
# 训练集AUC
probs_train= lr_model.predict_proba(X_train)
AUC1 = metrics.roc_auc_score(Y_train, probs_train[:,1])
print("Train Auc: %s"%(AUC1))
# 测试集AUC
probs_test= lr_model.predict_proba(X_test)
predict_test = lr_model.predict(X_test)
AUC2 = metrics.roc_auc_score(Y_test, probs_test[:,1])
print("Test Auc: %s"%(AUC2))
# 准确率
accuracy = metrics.accuracy_score(Y_test, predict_test)
print("Test Accuracy: %s"%(accuracy))
# 召回率
recall = metrics.recall_score(Y_test, predict_test)
print("Test Recall: %s"%(recall))
def test(args):
if args.enable_hvd:
import horovod.torch as hvd
hvd_size, hvd_rank, hvd_local_rank = utils.init_hvd_cuda(
args.enable_hvd, args.enable_gpu)
if args.load_ckpt_name is not None:
#TODO: choose ckpt_path
ckpt_path = utils.get_checkpoint(args.model_dir, args.load_ckpt_name)
else:
ckpt_path = utils.latest_checkpoint(args.model_dir)
assert ckpt_path is not None, 'No ckpt found'
checkpoint = torch.load(ckpt_path)
if 'subcategory_dict' in checkpoint:
subcategory_dict = checkpoint['subcategory_dict']
else:
subcategory_dict = {}
category_dict = checkpoint['category_dict']
word_dict = checkpoint['word_dict']
domain_dict = checkpoint['domain_dict']
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
config = AutoConfig.from_pretrained("bert-base-uncased",
output_hidden_states=True)
bert_model = AutoModel.from_pretrained("bert-base-uncased", config=config)
model = ModelBert(args, bert_model, len(category_dict), len(domain_dict),
len(subcategory_dict))
if args.enable_gpu:
model.cuda()
model.load_state_dict(checkpoint['model_state_dict'])
logging.info(f"Model loaded from {ckpt_path}")
if args.enable_hvd:
hvd.broadcast_parameters(model.state_dict(), root_rank=0)
model.eval()
torch.set_grad_enabled(False)
news, news_index, category_dict, domain_dict, subcategory_dict = read_news_bert(
os.path.join(args.root_data_dir,
f'{args.market}/{args.test_dir}/news.tsv'), args,
tokenizer)
news_title, news_title_type, news_title_attmask, \
news_abstract, news_abstract_type, news_abstract_attmask, \
news_body, news_body_type, news_body_attmask, \
news_category, news_domain, news_subcategory = get_doc_input_bert(
news, news_index, category_dict, domain_dict, subcategory_dict, args)
news_combined = np.concatenate([
x for x in
[news_title, news_title_type, news_title_attmask, \
news_abstract, news_abstract_type, news_abstract_attmask, \
news_body, news_body_type, news_body_attmask, \
news_category, news_domain, news_subcategory]
if x is not None], axis=1)
class NewsDataset(Dataset):
def __init__(self, data):
self.data = data
def __getitem__(self, idx):
return self.data[idx]
def __len__(self):
return self.data.shape[0]
def news_collate_fn(arr):
arr = torch.LongTensor(arr)
return arr
news_dataset = NewsDataset(news_combined)
news_dataloader = DataLoader(news_dataset,
batch_size=args.batch_size * 4,
num_workers=args.num_workers,
collate_fn=news_collate_fn)
news_scoring = []
with torch.no_grad():
for input_ids in tqdm(news_dataloader):
input_ids = input_ids.cuda()
news_vec = model.news_encoder(input_ids)
news_vec = news_vec.to(torch.device("cpu")).detach().numpy()
news_scoring.extend(news_vec)
news_scoring = np.array(news_scoring)
logging.info("news scoring num: {}".format(news_scoring.shape[0]))
dataloader = DataLoaderTest(
news_index=news_index,
news_scoring=news_scoring,
word_dict=word_dict,
news_bias_scoring=None,
data_dir=os.path.join(args.root_data_dir,
f'{args.market}/{args.test_dir}'),
filename_pat=args.filename_pat,
args=args,
world_size=hvd_size,
worker_rank=hvd_rank,
cuda_device_idx=hvd_local_rank,
enable_prefetch=True,
enable_shuffle=False,
enable_gpu=args.enable_gpu,
)
from metrics import roc_auc_score, ndcg_score, mrr_score, ctr_score
AUC = []
MRR = []
nDCG5 = []
nDCG10 = []
def print_metrics(hvd_local_rank, cnt, x):
logging.info("[{}] Ed: {}: {}".format(hvd_local_rank, cnt, \
'\t'.join(["{:0.2f}".format(i * 100) for i in x])))
def get_mean(arr):
return [np.array(i).mean() for i in arr]
#for cnt, (log_vecs, log_mask, news_vecs, news_bias, labels) in enumerate(dataloader):
for cnt, (log_vecs, log_mask, news_vecs, news_bias,
labels) in enumerate(dataloader):
his_lens = torch.sum(log_mask,
dim=-1).to(torch.device("cpu")).detach().numpy()
if args.enable_gpu:
log_vecs = log_vecs.cuda(non_blocking=True)
log_mask = log_mask.cuda(non_blocking=True)
user_vecs = model.user_encoder(log_vecs, log_mask).to(
torch.device("cpu")).detach().numpy()
for index, user_vec, news_vec, bias, label, his_len in zip(
range(len(labels)), user_vecs, news_vecs, news_bias, labels,
his_lens):
if label.mean() == 0 or label.mean() == 1:
continue
score = np.dot(news_vec, user_vec)
auc = roc_auc_score(label, score)
mrr = mrr_score(label, score)
ndcg5 = ndcg_score(label, score, k=5)
ndcg10 = ndcg_score(label, score, k=10)
AUC.append(auc)
MRR.append(mrr)
nDCG5.append(ndcg5)
nDCG10.append(ndcg10)
if cnt % args.log_steps == 0:
print_metrics(hvd_rank, cnt * args.batch_size,
get_mean([AUC, MRR, nDCG5, nDCG10]))
# stop scoring
dataloader.join()
for i in range(2):
print_metrics(hvd_rank, cnt * args.batch_size,
get_mean([AUC, MRR, nDCG5, nDCG10]))
def test(rank, args):
if rank is None:
is_distributed = False
rank = 0
else:
is_distributed = True
if is_distributed:
utils.setuplogger()
dist.init_process_group('nccl',
world_size=args.nGPU,
init_method='env://',
rank=rank)
torch.cuda.set_device(rank)
if args.load_ckpt_name is not None:
ckpt_path = utils.get_checkpoint(args.model_dir, args.load_ckpt_name)
assert ckpt_path is not None, 'No checkpoint found.'
checkpoint = torch.load(ckpt_path, map_location='cpu')
subcategory_dict = checkpoint['subcategory_dict']
category_dict = checkpoint['category_dict']
word_dict = checkpoint['word_dict']
dummy_embedding_matrix = np.zeros(
(len(word_dict) + 1, args.word_embedding_dim))
module = importlib.import_module(f'model.{args.model}')
model = module.Model(args, dummy_embedding_matrix, len(category_dict),
len(subcategory_dict))
model.load_state_dict(checkpoint['model_state_dict'])
logging.info(f"Model loaded from {ckpt_path}")
if args.enable_gpu:
model.cuda(rank)
model.eval()
torch.set_grad_enabled(False)
news, news_index = read_news(os.path.join(args.test_data_dir, 'news.tsv'),
args,
mode='test')
news_title, news_category, news_subcategory = get_doc_input(
news, news_index, category_dict, subcategory_dict, word_dict, args)
news_combined = np.concatenate([
x
for x in [news_title, news_category, news_subcategory] if x is not None
],
axis=-1)
news_dataset = NewsDataset(news_combined)
news_dataloader = DataLoader(news_dataset,
batch_size=args.batch_size,
num_workers=4)
news_scoring = []
with torch.no_grad():
for input_ids in tqdm(news_dataloader):
input_ids = input_ids.cuda(rank)
news_vec = model.news_encoder(input_ids)
news_vec = news_vec.to(torch.device("cpu")).detach().numpy()
news_scoring.extend(news_vec)
news_scoring = np.array(news_scoring)
logging.info("news scoring num: {}".format(news_scoring.shape[0]))
if rank == 0:
doc_sim = 0
for _ in tqdm(range(1000000)):
i = random.randrange(1, len(news_scoring))
j = random.randrange(1, len(news_scoring))
if i != j:
doc_sim += np.dot(news_scoring[i], news_scoring[j]) / (
np.linalg.norm(news_scoring[i]) *
np.linalg.norm(news_scoring[j]))
logging.info(f'News doc-sim: {doc_sim / 1000000}')
data_file_path = os.path.join(args.test_data_dir, f'behaviors_{rank}.tsv')
def collate_fn(tuple_list):
log_vecs = torch.FloatTensor([x[0] for x in tuple_list])
log_mask = torch.FloatTensor([x[1] for x in tuple_list])
news_vecs = [x[2] for x in tuple_list]
labels = [x[3] for x in tuple_list]
return (log_vecs, log_mask, news_vecs, labels)
dataset = DatasetTest(data_file_path, news_index, news_scoring, args)
dataloader = DataLoader(dataset,
batch_size=args.batch_size,
collate_fn=collate_fn)
from metrics import roc_auc_score, ndcg_score, mrr_score
AUC = []
MRR = []
nDCG5 = []
nDCG10 = []
def print_metrics(rank, cnt, x):
logging.info("[{}] {} samples: {}".format(
rank, cnt, '\t'.join(["{:0.2f}".format(i * 100) for i in x])))
def get_mean(arr):
return [np.array(i).mean() for i in arr]
def get_sum(arr):
return [np.array(i).sum() for i in arr]
local_sample_num = 0
for cnt, (log_vecs, log_mask, news_vecs, labels) in enumerate(dataloader):
local_sample_num += log_vecs.shape[0]
if args.enable_gpu:
log_vecs = log_vecs.cuda(rank, non_blocking=True)
log_mask = log_mask.cuda(rank, non_blocking=True)
user_vecs = model.user_encoder(log_vecs, log_mask).to(
torch.device("cpu")).detach().numpy()
for user_vec, news_vec, label in zip(user_vecs, news_vecs, labels):
if label.mean() == 0 or label.mean() == 1:
continue
score = np.dot(news_vec, user_vec)
auc = roc_auc_score(label, score)
mrr = mrr_score(label, score)
ndcg5 = ndcg_score(label, score, k=5)
ndcg10 = ndcg_score(label, score, k=10)
AUC.append(auc)
MRR.append(mrr)
nDCG5.append(ndcg5)
nDCG10.append(ndcg10)
if cnt % args.log_steps == 0:
print_metrics(rank, local_sample_num,
get_mean([AUC, MRR, nDCG5, nDCG10]))
logging.info('[{}] local_sample_num: {}'.format(rank, local_sample_num))
if is_distributed:
local_sample_num = torch.tensor(local_sample_num).cuda(rank)
dist.reduce(local_sample_num, dst=0, op=dist.ReduceOp.SUM)
local_metrics_sum = torch.FloatTensor(
get_sum([AUC, MRR, nDCG5, nDCG10])).cuda(rank)
dist.reduce(local_metrics_sum, dst=0, op=dist.ReduceOp.SUM)
if rank == 0:
print_metrics('*', local_sample_num,
local_metrics_sum / local_sample_num)
else:
print_metrics('*', local_sample_num,
get_mean([AUC, MRR, nDCG5, nDCG10]))