def process_meta(file):
fi = open(datafilename(category, file), "r")
fo = open(datafilename(category, "item-info"), "w")
for line in fi:
obj = eval(line)
cat = obj["categories"][0][-1]
print >> fo, obj["asin"] + "\t" + cat
def process_reviews(file):
fi = open(datafilename(category, file), "r")
user_map = {}
fo = open(datafilename(category, "reviews-info"), "w")
for line in fi:
obj = eval(line)
userID = obj["reviewerID"]
itemID = obj["asin"]
rating = obj["overall"]
time = obj["unixReviewTime"]
print >> fo, userID + "\t" + itemID + "\t" + str(rating) + "\t" + str(
time)
def split_test_by_time(cut_time):
fi = open(datafilename(category, "local_all_sample_by_time"), "r")
ftrain = open(datafilename(category, "local_train_by_time"), "w")
ftest = open(datafilename(category, "local_test_by_time"), "w")
for line in fi:
line = line.strip()
time = float(line.split("\t")[-1])
if time <= cut_time:
print >> ftrain, line[:-2]
else:
print >> ftest, line[:-2]
def get_cut_timestamp(train_percent=0.85):
time_list = []
fi = open(datafilename(category, "local_all_sample_by_time"), "r")
path = datafilename(category, "local_all_sample_by_time")
samples_count = file_len(path)
train_size = int(samples_count * train_percent)
for line in fi:
line = line.strip()
time = float(line.split("\t")[-1])
time_list.append(time)
index = np.argsort(time_list, axis=-1)
cut_time_index = index[train_size]
return time_list[cut_time_index]
def get_all_samples():
fin = open(datafilename(category, "jointed-new-by-time"), "r")
fall = open(datafilename(category, "local_all_sample_by_time"), "w")
gap = np.array(
[1.1, 1.4, 1.7, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096])
last_user = "******"
line_idx = 0
for line in fin:
items = line.strip().split("\t")
clk = int(items[0])
user = items[1]
movie_id = items[2]
dt = items[4]
cat1 = items[5]
user_list = items[6]
user_t_list = items[7]
if user != last_user:
movie_id_list = []
cate1_list = []
movie_id_t_list = []
else:
history_clk_num = len(movie_id_list)
cat_str = ""
mid_str = ""
for c1 in cate1_list:
cat_str += c1 + ""
for mid in movie_id_list:
mid_str += mid + ""
dt_gap = []
for t in movie_id_t_list:
temp = float(dt) / 3600.0 / 24.0 - float(
t) / 3600.0 / 24.0 + 1.
dt_gap.append(str(np.sum(temp >= gap)))
dt_gap_str = "".join(dt_gap)
if len(cat_str) > 0: cat_str = cat_str[:-1]
if len(mid_str) > 0: mid_str = mid_str[:-1]
if history_clk_num >= 1: # 8 is the average length of user behavior
print >> fall, items[0] + "\t" + user + "\t" + movie_id + "\t" + cat1 + "\t" + mid_str + "\t" + cat_str + \
"\t" + user_list + "\t" + user_t_list + '\t' + dt_gap_str + "\t" + dt
last_user = user
if clk:
movie_id_list.append(movie_id)
cate1_list.append(cat1)
movie_id_t_list.append(dt)
line_idx += 1
def split_test_by_seqlen():
fi = open(datafilename(category, "local_test_by_time"), "r")
ftest_u1 = open(datafilename(category, "local_test_u1"), "w")
ftest_u2 = open(datafilename(category, "local_test_u2"), "w")
ftest_u3 = open(datafilename(category, "local_test_u3"), "w")
for line in fi:
line = line.strip()
item_seq = line.split("\t")[4]
sl = len(item_seq.split(""))
if sl < 5:
print >> ftest_u1, line
elif sl < 15:
print >> ftest_u2, line
else:
print >> ftest_u3, line
import cPickle
import random
import numpy as np
category = 'Amazon_Clothing_Shoes_and_Jewelry'
from path import datafilename
np.random.seed(1234)
random.seed(1234)
f_train = open(datafilename(category, "local_train_by_time"), "r").readlines()
f_test = open(datafilename(category, "local_test_by_time"), "r").readlines()
f_all = f_train + f_test
uid_dict = {}
mid_dict = {}
cat_dict = {}
iddd = 0
for line in f_all:
arr = line.strip("\n").split("\t")
clk = arr[0]
uid = arr[1]
mid = arr[2]
cat = arr[3]
mid_list = arr[4]
cat_list = arr[5]
if uid not in uid_dict:
def manual_join():
f_rev = open(datafilename(category, "reviews-info"), "r")
user_map = {} ## User clicked on the list of items
item_list = [] # all items list
useridToClickItem = {} # The user dict who clicked on the item
for line in f_rev:
line = line.strip()
items = line.split("\t")
# loctime = time.localtime(float(items[-1]))
# items[-1] = time.strftime('%Y-%m-%d', loctime)
if items[0] not in user_map:
user_map[items[0]] = []
user_map[items[0]].append(("\t".join(items), float(items[-1])))
item_list.append(items[1])
# The user dict who clicked on the item
f_rev = open(datafilename(category, "reviews-info"), "r")
for line in f_rev:
data = line.split("\t")
if data[1] not in useridToClickItem:
useridToClickItem[data[1]] = []
useridToClickItem[data[1]].append((data[0], float(data[-1])))
f_meta = open(datafilename(category, "item-info"), "r")
meta_map = {} # itemID map cate
for line in f_meta:
arr = line.strip().split("\t")
if arr[0] not in meta_map:
meta_map[arr[0]] = arr[1]
arr = line.strip().split("\t")
fo = open(datafilename(category, "jointed-new-by-time"), "w")
for key in user_map:
sorted_user_bh = sorted(user_map[key], key=lambda x: x[1])
for line, t in sorted_user_bh:
items = line.split("\t")
asin = items[1]
cur_t = float(items[3]) // 3600 // 24
j = 0
target_user_pos_in_seq = 0
while True:
asin_neg_index = random.randint(0, len(item_list) - 1)
asin_neg = item_list[asin_neg_index]
if asin_neg == asin:
continue
items[1] = asin_neg
if len(useridToClickItem[asin_neg]) == 0:
user_str = "default_user"
user_t_str = "-1"
else:
user_str = ""
user_t_str = ""
sorted_user_in_item_seq = sorted(
useridToClickItem[asin_neg], key=lambda x: x[1])
for i, (u, t) in enumerate(sorted_user_in_item_seq):
if int(t) > int(items[-1]):
target_user_pos_in_seq = i + 1
break
if u == items[0]:
continue
user_str += u + ""
user_t = float(cur_t) - t // 3600 // 24 + 1.
user_t_str += str(np.sum(user_t >= gap)) + ""
if len(user_str) > 0:
user_str = user_str[:-1]
user_t_str = user_t_str[:-1]
if len(user_str) == 0:
user_str = "default_user"
user_t_str = "-1"
if asin_neg in meta_map:
print >> fo, "0" + "\t" + "\t".join(
items
) + "\t" + meta_map[
asin_neg] + "\t" + user_str + "\t" + user_t_str + "\t" + items[
3]
else:
print >> fo, "0" + "\t" + "\t".join(
items
) + "\t" + "default_cat" + "\t" + user_str + "\t" + user_t_str + "\t" + items[
3]
j += 1
if j == 1: # negative sampling frequency
break
target_user_pos_in_seq = 0
# useridToClickItem[asin][0].remove(items[0])
if len(useridToClickItem[asin]) == 0:
user_str = "default_user"
user_t_str = "-1"
else:
user_str = ""
user_t_str = ""
sorted_user_in_item_seq = sorted(useridToClickItem[asin],
key=lambda x: x[1])
for i, (u, t) in enumerate(sorted_user_in_item_seq):
if int(t) > int(items[-1]):
target_user_pos_in_seq = i
break
if u == items[0]:
continue
user_str += u + ""
user_t = float(cur_t) - t // 3600 // 24 + 1.
user_t_str += str(np.sum(user_t >= gap)) + ""
if len(user_str) > 0:
user_str = user_str[:-1]
user_t_str = user_t_str[:-1]
if len(user_str) == 0:
user_str = "default_user"
user_t_str = "-1"
if asin in meta_map:
print >> fo, "1" + "\t" + line + "\t" + meta_map[
asin] + "\t" + user_str + "\t" + user_t_str + "\t" + items[
3]
else:
print >> fo, "1" + "\t" + line + "\t" + "default_cat" + "\t" + user_str + "\t" + user_t_str + "\t" + items[
3]