def setUp(self):
if list(ScoreModel.where(userid="1998080911").select()) == []:
create_score_instance(
"1998080911",
"0000000001",
"2018-12-1",
5,
u"电影肥肠好看哈哈哈哈哈哈",
1,
4
)
if list(ScoreModel.where(userid="1999022511").select()) == []:
create_score_instance(
"1999022511",
"0000000002",
"2018-12-7",
14,
u"还阔以，嘻嘻",
1,
3
)
if list(UserModel.where(userid="1999080611").select()) == []:
create_score_instance(
"1999080611",
"0000000003",
"2018-12-4",
60,
u"龙猫太可爱了",
1,
5
)
def test_update(self):
ScoreModel.where(userid="1999080611").update(star=2)
score = ScoreModel.where(userid="1999080611").select().next()
self.assertEquals(score.userid, "1999080611")
self.assertEquals(score.movieid, "0000000003")
self.assertEquals(score.comDate, "2018-12-4")
self.assertEquals(score.value, 60)
self.assertEquals(score.content, u"龙猫太可爱了")
self.assertEquals(score.watched, 1)
self.assertEquals(score.star, 2)
def test_instance(self):
score = ScoreModel.where(userid="1998080911").select().next()
self.assertEquals(score.userid, "1998080911")
self.assertEquals(score.movieid, "0000000001")
self.assertEquals(score.comDate, "2018-12-1")
self.assertEquals(score.value, 5)
self.assertEquals(score.content, u"电影肥肠好看哈哈哈哈哈哈")
self.assertEquals(score.watched, 1)
self.assertEquals(score.star, 4)
def post(self):
current_time = datetime.datetime.now()
user_id = self.get_json_argument('user_id')
activity_id = self.get_json_argument('activity_id')
score = self.get_json_argument('score')
comment = self.get_json_argument('comment')
score = ScoreModel(user_id=user_id, activity_id=activity_id, score=score, comment=comment, is_del=False,
create_time=current_time, update_time=current_time)
self.model_config.add(score)
return {'score_id': score.id}
def test_select(self):
scores = ScoreModel.where().select()
for score in scores:
if score.userid == "1998080911": pass
elif score.userid == "1999022511": pass
else:
self.assertEquals(score.userid, "1999080611")
self.assertEquals(score.movieid, "0000000003")
self.assertEquals(score.comDate, "2018-12-4")
self.assertEquals(score.value, 60)
self.assertEquals(score.content, u"龙猫太可爱了")
self.assertEquals(score.watched, 1)
self.assertEquals(score.star, 5)
device = torch.device("cuda", local_rank)
checkpoints_dir = './checkpoints'
start_epoch = 0
use_bert = True
if not os.path.exists(checkpoints_dir) and local_rank == 0:
os.makedirs(checkpoints_dir)
kdd_dataset = Dataset(use_bert=use_bert)
sampler = DistributedSampler(kdd_dataset)
loader = DataLoader(kdd_dataset,
collate_fn=collate_fn,
batch_size=150,
sampler=sampler,
num_workers=20)
nhead = 4
score_model = ScoreModel(kdd_dataset.unknown_token + 1,
1024,
1024,
use_bert=use_bert).cuda()
image_encoder = ImageEncoder(input_dim=2048, output_dim=1024, nhead=nhead)
image_encoder.load_pretrained_weights(
path='../user_data/image_encoder_large.pth')
image_encoder = image_encoder.cuda()
# text_generator = TextGenerator(score_model.embed.num_embeddings).cuda()
# score_model = ScoreModel(30522, 256, num_heads=1).cuda()
# category_embedding = CategoryEmbedding(256).cuda()
optimizer = Adam(score_model.get_params() + image_encoder.get_params())
if start_epoch > 0 and local_rank == 0:
checkpoints = torch.load(
os.path.join(checkpoints_dir,
'model-epoch{}.pth'.format(start_epoch)))
sampler = DistributedSampler(kdd_dataset)
loader = DataLoader(kdd_dataset,
collate_fn=collate_fn,
batch_size=130,
sampler=sampler,
num_workers=15)
nhead = 4
text_encoder = TextEncoder(kdd_dataset.unknown_token + 1,
1024,
256,
use_bert=use_bert).cuda()
image_encoder = ImageEncoder(input_dim=2048, output_dim=1024, nhead=nhead)
image_encoder.load_pretrained_weights(
path='../user_data/image_encoder_large.pth')
image_encoder = image_encoder.cuda()
score_model = ScoreModel(1024, 256).cuda()
# text_generator = TextGenerator(text_encoder.embed.num_embeddings).cuda()
# score_model = ScoreModel(30522, 256, num_heads=1).cuda()
# category_embedding = CategoryEmbedding(256).cuda()
optimizer = Adam(image_encoder.get_params() + text_encoder.get_params() +
score_model.get_params())
if start_epoch > 0 and local_rank == 0:
checkpoints = torch.load(
os.path.join(checkpoints_dir,
'model-epoch{}.pth'.format(start_epoch)), 'cpu')
text_encoder.load_state_dict(checkpoints['query'])
image_encoder.load_state_dict(checkpoints['item'])
score_model.load_state_dict(checkpoints['score'])
# text_generator.load_state_dict(checkpoints['generator'])
from matplotlib.ticker import LinearLocator
from matplotlib import cm
def marginal_prob_std(t, D):
t = torch.tensor(t, device=device)
return torch.sqrt(D * (1 - np.exp(-2 * t)))
D = 1
marginal_prob_std_fn = functools.partial(marginal_prob_std, D=D)
weights = torch.load('model_weights.pth')
dim = 1
embed_dim = 128
score = ScoreModel(dim, embed_dim, marginal_prob_std_fn)
# model = autoencoder(dim,embed_dim,marginal_prob_std_fn)
score.load_state_dict(weights)
device = 'cpu'
fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
# Make data.
x = np.arange(-5, 5, 0.1)
t = np.arange(1e-3, 1, 0.1)
x, t = np.meshgrid(x, t)
r = np.sqrt(x**2 + t**2)
numrow = 0
def valid(epoch=1,
checkpoints_dir='./checkpoints',
use_bert=False,
data_path=None,
out_path='../prediction_result/valid_pred.json',
output_ndcg=True):
print("valid epoch{}".format(epoch))
if data_path is not None:
kdd_dataset = ValidDataset(data_path, use_bert=use_bert)
else:
kdd_dataset = ValidDataset(data_path, use_bert=use_bert)
loader = DataLoader(kdd_dataset,
collate_fn=collate_fn_valid,
batch_size=128,
shuffle=False,
num_workers=8)
tbar = tqdm(loader)
text_encoder = TextEncoder(kdd_dataset.unknown_token + 1,
1024,
256,
use_bert=use_bert).cuda()
image_encoder = ImageEncoder(input_dim=2048, output_dim=1024,
nhead=4).cuda()
score_model = ScoreModel(1024, 256).cuda()
# category_embedding = model.CategoryEmbedding(768).cuda()
checkpoints = torch.load(
os.path.join(checkpoints_dir, 'model-epoch{}.pth'.format(epoch)))
text_encoder.load_state_dict(checkpoints['query'])
image_encoder.load_state_dict(checkpoints['item'])
score_model.load_state_dict(checkpoints['score'])
# score_model.load_state_dict(checkpoints['score'])
outputs = {}
image_encoder.eval()
text_encoder.eval()
score_model.eval()
for query_id, product_id, query, query_len, features, boxes, category, obj_len in tbar:
query, query_len = query.cuda(), query_len.cuda()
query, hidden = text_encoder(query, query_len)
features, boxes, obj_len = features.cuda(), boxes.cuda(), obj_len.cuda(
)
features = image_encoder(features, boxes, obj_len)
score = score_model(query, hidden, query_len, features)
score = score.data.cpu().numpy()
# print(score2)
for q_id, p_id, s in zip(query_id.data.numpy(),
product_id.data.numpy(), score):
outputs.setdefault(str(q_id), [])
outputs[str(q_id)].append((p_id, s))
for k, v in outputs.items():
v = sorted(v, key=lambda x: x[1], reverse=True)
v = [(str(x[0]), float(x[1])) for x in v]
outputs[k] = v
with open(out_path, 'w') as f:
json.dump(outputs, f)
if output_ndcg:
pred = read_json(out_path)
gt = read_json('../data/valid/valid_answer.json')
score = 0
k = 5
for key, val in gt.items():
ground_truth_ids = [str(x) for x in val]
predictions = [x[0] for x in pred[key][:k]]
ref_vec = [1.0] * len(ground_truth_ids)
pred_vec = [
1.0 if pid in ground_truth_ids else 0.0 for pid in predictions
]
score += get_ndcg(pred_vec, ref_vec, k)
# print(key)
# print([pid for pid in predictions if pid not in ground_truth_ids])
# print('========')
# score += len(set(predictions).intersection(ground_truth_ids)) / len(ground_truth_ids)
score = score / len(gt)
print('ndcg@%d: %.4f' % (k, score))
return score
else:
return None
loss = torch.mean(torch.sum((score * std[:, None] + z)**2, dim=1)) return loss D = 1 marginal_prob_std_fn = functools.partial(marginal_prob_std, D=D) diffusion_coeff_fn = functools.partial(diffusion_coeff, D=D) # Define model embed_dim = 128 # Dimension time is embedded in dim = 1 model = ScoreModel(dim,embed_dim = embed_dim,marginal_prob_std=marginal_prob_std_fn) # model = autoencoder(dim,embed_dim,marginal_prob_std_fn) # Define optimizer lr = 1e-4 momentum = 0.9 optimizer = optim.SGD(model.parameters(), lr=lr, momentum = momentum) n_epochs = 5000 batch_size = 1024 losses = [] # torch.autograd.set_detect_anomaly(True) for epoch in range(n_epochs):
def test_count(self):
cnt = ScoreModel.where(content=u"电影肥肠好看哈哈哈哈哈哈").count()
self.assertEquals(1, cnt)
def test_limit(self):
scores = ScoreModel.where(content=u"电影肥肠好看哈哈哈哈哈哈").limit(2).select()
cnt = 0
for score in scores: cnt += 1
self.assertEquals(1, cnt)
def create_score_instance(
userid,
movieid,
comDate,
value,
content,
watched,
star
):
score = ScoreModel()
score.userid = userid
score.movieid = movieid
score.comDate = comDate
score.value = value
score.content = content
score.watched = watched
score.star = star
score.save()
return score