def test_save_and_load_model(self): model = TensorRec(n_components=10) model.fit(self.interactions, self.user_features, self.item_features, epochs=10) predictions = model.predict(user_features=self.user_features, item_features=self.item_features) ranks = model.predict_rank(user_features=self.user_features, item_features=self.item_features) model.save_model(directory_path=self.test_dir) # Check that, after saving, the same predictions come back predictions_after_save = model.predict(user_features=self.user_features, item_features=self.item_features) ranks_after_save = model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertTrue((predictions == predictions_after_save).all()) self.assertTrue((ranks == ranks_after_save).all()) # Blow away the session set_session(None) tf.reset_default_graph() # Reload the model, predict, and check for equal predictions new_model = TensorRec.load_model(directory_path=self.test_dir) new_predictions = new_model.predict(user_features=self.user_features, item_features=self.item_features) new_ranks = new_model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertTrue((predictions == new_predictions).all()) self.assertTrue((ranks == new_ranks).all())
def test_save_and_load_model(self): model = TensorRec(n_components=10) model.fit(self.interactions, self.user_features, self.item_features, epochs=10) predictions = model.predict(user_features=self.user_features, item_features=self.item_features) ranks = model.predict_rank(user_features=self.user_features, item_features=self.item_features) model.save_model(directory_path=self.test_dir) # Check that, after saving, the same predictions come back predictions_after_save = model.predict( user_features=self.user_features, item_features=self.item_features) ranks_after_save = model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertTrue((predictions == predictions_after_save).all()) self.assertTrue((ranks == ranks_after_save).all()) # Blow away the session set_session(None) tf.reset_default_graph() # Reload the model, predict, and check for equal predictions new_model = TensorRec.load_model(directory_path=self.test_dir) new_predictions = new_model.predict(user_features=self.user_features, item_features=self.item_features) new_ranks = new_model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertTrue((predictions == new_predictions).all()) self.assertTrue((ranks == new_ranks).all())
def setUpClass(cls): cls.interactions, cls.user_features, cls.item_features = generate_dummy_data_with_indicator( num_users=10, num_items=12, interaction_density=.5) model = TensorRec(n_components=10) model.fit(cls.interactions, cls.user_features, cls.item_features, epochs=10) cls.model = model cls.ranks = model.predict_rank(user_features=cls.user_features, item_features=cls.item_features)
def metric_test(self): """ uses tensorrec eval as benchmark for rating performance of various reco algorithms """ k = 10 latent_factor = 10 n_users = 10 n_items = 12 interactions, user_features, item_features = util.generate_dummy_data_with_indicator( num_users=n_users, num_items=n_items, interaction_density=.5) print("interactiosn shape={}".format(np.shape(interactions))) print("user features shape={}".format(np.shape( user_features.toarray()))) print("item features shape={}".format(np.shape( item_features.toarray()))) model = TensorRec(n_components=latent_factor) model.fit(interactions, user_features, item_features, epochs=19) ranks = model.predict_rank(user_features=user_features, item_features=item_features) print("Ranks shape={}".format(np.shape(ranks))) self.assertTrue(np.shape(interactions) == np.shape(ranks)) tr_recall_result = eval.recall_at_k(predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print (tr_recall_result.mean()) tr_precision_result = eval.precision_at_k( predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print(tr_precision_result.mean()) # we need csr for interactions data interactions_ = interactions.tocsr() recall_result = metrics.recall_at_k(ranks, interactions_, k=k, preserve_rows=False) # print(recall_result.mean()) precision_result = metrics.precision_at_k(ranks, interactions_, k=k, preserve_rows=False) # print (precision_result.mean()) self.assertTrue(tr_recall_result.mean() == recall_result.mean()) self.assertTrue(tr_precision_result.mean() == precision_result.mean())
def setUpClass(cls): cls.interactions, cls.user_features, cls.item_features = generate_dummy_data_with_indicator( num_users=10, num_items=12, interaction_density=.5) model = TensorRec(n_components=10) model.fit(cls.interactions, cls.user_features, cls.item_features, epochs=10) cls.model = model cls.ranks = model.predict_rank(user_features=cls.user_features, item_features=cls.item_features)
def test_predict_fail_unfit(self): model = TensorRec() with self.assertRaises(ModelNotFitException): model.predict(self.user_features, self.item_features) with self.assertRaises(ModelNotFitException): model.predict_rank(self.user_features, self.item_features) with self.assertRaises(ModelNotFitException): model.predict_user_representation(self.user_features) with self.assertRaises(ModelNotFitException): model.predict_item_representation(self.item_features) with self.assertRaises(ModelNotFitException): model.predict_user_attention_representation(self.user_features) with self.assertRaises(ModelNotFitException): model.predict_similar_items(self.item_features, item_ids=[1], n_similar=5) with self.assertRaises(ModelNotFitException): model.predict_item_bias(self.item_features) with self.assertRaises(ModelNotFitException): model.predict_user_bias(self.user_features)
def test_save_and_load_model_same_session(self): model = TensorRec(n_components=10) model.fit(self.interactions, self.user_features, self.item_features, epochs=10) predictions = model.predict(user_features=self.user_features, item_features=self.item_features) ranks = model.predict_rank(user_features=self.user_features, item_features=self.item_features) model.save_model(directory_path=self.test_dir) # Reload the model, predict, and check for equal predictions new_model = TensorRec.load_model(directory_path=self.test_dir) new_predictions = new_model.predict(user_features=self.user_features, item_features=self.item_features) new_ranks = new_model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertEqual(predictions.all(), new_predictions.all()) self.assertEqual(ranks.all(), new_ranks.all())
def test_save_and_load_model_same_session(self): model = TensorRec(n_components=10) model.fit(self.interactions, self.user_features, self.item_features, epochs=10) predictions = model.predict(user_features=self.user_features, item_features=self.item_features) ranks = model.predict_rank(user_features=self.user_features, item_features=self.item_features) model.save_model(directory_path=self.test_dir) # Reload the model, predict, and check for equal predictions new_model = TensorRec.load_model(directory_path=self.test_dir) new_predictions = new_model.predict(user_features=self.user_features, item_features=self.item_features) new_ranks = new_model.predict_rank(user_features=self.user_features, item_features=self.item_features) self.assertTrue((predictions == new_predictions).all()) self.assertTrue((ranks == new_ranks).all())
ax.scatter(*zip(*movie_positions[movies_to_plot]), s=2) ax.set_aspect('equal') for i, movie in enumerate(movies_to_plot): movie_name = item_titles[movie] movie_position = movie_positions[movie] # Comment this line to remove movie titles to the plot. ax.annotate(movie_name, movie_position[0:2], fontsize='x-small') file = '/tmp/tensorrec/movielens/epoch_{}.jpg'.format(epoch) plt.savefig(file) logging.info("Finished epoch {}".format(epoch)) ranks = model.predict_rank( user_features=user_features, item_features=item_features, ) p_at_k = precision_at_k(ranks, test_interactions, k=5) r_at_k = recall_at_k(ranks, test_interactions, k=30) logging.info("Precision@5: {}, Recall@30: {}".format(np.mean(p_at_k), np.mean(r_at_k))) # Use the collected JPG files to create an MP4 video of the model fitting, then delete the JPGs. fps = 12 file_list = glob.glob('/tmp/tensorrec/movielens/*.jpg') list.sort(file_list, key=lambda x: int(x.split('_')[1].split('.jpg')[0])) clip = mpy.ImageSequenceClip(file_list, fps=fps) vid_file = '/tmp/tensorrec/movielens/movielens.mp4' clip.write_videofile(filename=vid_file, fps=fps,
ax.scatter(*zip(*user_positions[user_to_plot]), color='r', s=1) ax.scatter(*zip(*movie_positions[movies_to_plot]), s=2) ax.set_aspect('equal') for i, movie in enumerate(movies_to_plot): movie_name = item_titles[movie] movie_position = movie_positions[movie] # Comment this line to remove movie titles to the plot. ax.annotate(movie_name, movie_position[0:2], fontsize='x-small') file = '/tmp/tensorrec/movielens/epoch_{}.jpg'.format(epoch) plt.savefig(file) logging.info("Finished epoch {}".format(epoch)) ranks = model.predict_rank(user_features=user_features, item_features=item_features,) p_at_k = precision_at_k(ranks, test_interactions, k=5) r_at_k = recall_at_k(ranks, test_interactions, k=30) logging.info("Precision@5: {}, Recall@30: {}".format(np.mean(p_at_k), np.mean(r_at_k))) # Use the collected JPG files to create an MP4 video of the model fitting, then delete the JPGs. fps = 12 file_list = glob.glob('/tmp/tensorrec/movielens/*.jpg') list.sort(file_list, key=lambda x: int(x.split('_')[1].split('.jpg')[0])) clip = mpy.ImageSequenceClip(file_list, fps=fps) vid_file = '/tmp/tensorrec/movielens/movielens.mp4' clip.write_videofile(filename=vid_file, fps=fps, codec='mpeg4', preset='veryslow', ffmpeg_params=['-qscale:v', '10']) for file in file_list: os.remove(file)
def main(): # 데이터 로드 ## 마스터 데이터(상호 작용) masterdf = pd.read_csv('./data/Transactions.csv') masterdf.columns = ['Transaction ID', 'Customer ID', 'Transaction Date', 'Prod Subcat Code', 'Prod Cat Code', 'Qty', 'Rate', 'Tax', 'Total Amt', 'Store Type'] # 데이터 정리 및 표준화를 위해 데이터 열 명칭 변경 masterdf['Store Type Code'] = pd.factorize(masterdf['Store Type'])[0] # 상점 코드 타입을 숫자형으로 변경하여 새 열에 저장 masterdf['Date'] = pd.DatetimeIndex(masterdf['Transaction Date'], dayfirst=True).date # 거래 날짜를 pandas의 datetime index로 표준화 masterdf['Net Sales'] = masterdf['Qty'] * masterdf['Rate'] # quantity와 based price에서 총 순 매출액(Net sales) 계산 (도시마다의 세금이 다를 수 있어 세금 제외) masterdf['Material'] = masterdf['Prod Cat Code'].astype(str) + '-' + masterdf['Prod Subcat Code'].astype(str) + '-' + masterdf['Store Type'].astype(str) # category, subcategory, store type을 이용하여 고유한 material 표시기를 생성 masterdf[['Prod Cat Code','Prod Subcat Code', 'Store Type', 'Material']].drop_duplicates(subset='Material') ## 소비자 데이터(소비자 특성) custdf = pd.read_csv('./data/Customer.csv') custdf.columns = ['Customer ID', 'DOB', 'Gender', 'City Code'] ## 아이탬 특징 데이터 skudf = pd.read_csv('./data/prod_cat_info.csv') skudf.columns = ['Prod Cat Code', 'Prod Cat', 'Prod Sub Cat Code', 'Prod Subcat'] # 데이터 생성 ## RECENCY (최신성) recency_df = masterdf.groupby('Customer ID').Date.max().reset_index() recency_df.columns = ['Customer ID','Last Purchase'] recency_df['Recency'] = recency_df['Last Purchase'].apply(lambda x: (now - x).days) recency_df = recency_df[['Customer ID', 'Recency']] ## FREQUENCY (빈도) frequency_df = masterdf.groupby('Customer ID')['Date'].count().reset_index() frequency_df.columns = ['Customer ID','Frequency'] ## MONETARY (금액) monetary_df = masterdf.groupby('Customer ID')['Net Sales'].sum().reset_index() monetary_df.columns = ['Customer ID','Monetary'] ## VARIETY (종류) variety_df = masterdf.groupby('Customer ID')['Material'].nunique().reset_index() variety_df.columns = ['Customer ID','Variety'] ## RFMV rfmv = recency_df.copy() rfmv = rfmv.merge(frequency_df, on='Customer ID') rfmv = rfmv.merge(monetary_df, on='Customer ID') rfmv = rfmv.merge(variety_df, on='Customer ID') rfmv_quantiles = rfmv.iloc[:, 1:].quantile(q = [0.25, 0.5, 0.75]).to_dict() # R, F, M, V의 25%, 50%, 75%의 사분위수를 dictonary 형식으로 저장 rfmv2 = rfmv.copy() rfmv2['R_q'] = rfmv2['Recency'].apply(RecencyScore, args=('Recency', rfmv_quantiles )) rfmv2['F_q'] = rfmv2['Frequency'].apply(FMVScore, args=('Frequency', rfmv_quantiles )) rfmv2['M_q'] = rfmv2['Monetary'].apply(FMVScore, args=('Monetary', rfmv_quantiles )) rfmv2['V_q'] = rfmv2['Variety'].apply(FMVScore, args=('Variety', rfmv_quantiles )) rfmv2 = rfmv2[['Customer ID', 'R_q', 'F_q', 'M_q', 'V_q',]] ## 각 구성 요소의 총 점수 합계 rfmv2['Total_Score'] = rfmv2['R_q'] + rfmv2['F_q'] + rfmv2['M_q'] + rfmv2['V_q'] rfmv2 = rfmv2[['Customer ID', 'Total_Score']] # 중요(IMPORTANT) : 인덱스를 고객 번호로 설정 rfmv2.index = rfmv2['Customer ID'] rfmv2 = rfmv2.drop('Customer ID', 1) # 최적의 군집 수를 찾기 위해 elbow 방식 (차후 이 과정을 조정할 필요가 있음) wcss = [] for i in range(2,10): kmeans = KMeans(n_clusters=i, init='k-means++') kmeans.fit(rfmv2) wcss.append(kmeans.inertia_) # 위 "elbow" 그래프의 최적의 수를 이용하여 KMean 군집 적용 kmeans = KMeans(n_clusters=4, init='random', random_state=None) clusters = kmeans.fit_predict(rfmv2) ### 군집 결과를 원본 rfmv 데이터에 추가 rfmv['Clusters'] = clusters # Recommendation Weight active_cust = rfmv[rfmv.Recency < 365] # 최근 1년(365일)을 기준으로 하여 실고객에게 추천 cleaned_df = masterdf.merge(active_cust[['Customer ID','Clusters']], how='left', on='Customer ID') # 군집화된 고객 특징을 마스터 데이터에 결합 cleaned_df = cleaned_df[cleaned_df['Clusters'].notnull()] # 군집을 기준으로 null 값이 존재하는 행 삭제 cleaned_df = cleaned_df.merge(custdf[['Customer ID', 'City Code']], how='left', on='Customer ID') ## 소비자 데이터 추가 cleaned_df = cleaned_df.merge(skudf[['Prod Cat', 'Prod Cat Code']], how='left', on='Prod Cat Code') # sku 특징(물품 카테고리) 를 마스터 데이터에 결합 # 필수 열 가져오기 final_cleaned_df = cleaned_df final_cleaned_df = final_cleaned_df[['Prod Cat','Material','Qty','Customer ID','Clusters',]] # 고유한 고객 목록 유지, 중복 제거 cust_grouped = final_cleaned_df.groupby(['Customer ID', 'Prod Cat', 'Material', 'Clusters']).sum().reset_index() ## Interaction Matrix interactions = cust_grouped.groupby(['Customer ID', 'Material'])['Qty'].sum().unstack().fillna(0) minmaxscaler = preprocessing.MinMaxScaler() interactions_scaled = minmaxscaler.fit_transform(interactions) interactions_scaled = pd.DataFrame(interactions_scaled) interactions_scaled.index = interactions.index interactions_scaled.columns = interactions.columns ## User Features Matrix cust_qty = cust_grouped.groupby(['Customer ID', 'Prod Cat'])['Qty'].sum().unstack().fillna(0) minmaxscaler = preprocessing.MinMaxScaler() cust_qty_scaled = minmaxscaler.fit_transform(cust_qty) cust_qty_scaled = pd.DataFrame(cust_qty_scaled) cust_qty_scaled.index = cust_qty.index cust_qty_scaled.columns = cust_qty.columns cust_clus = cust_grouped.groupby(['Customer ID', 'Clusters'])['Clusters'].nunique().unstack().fillna(0) customer_features = pd.merge(cust_qty_scaled, cust_clus, left_index=True, right_index=True, how='inner') customer_features = customer_features.rename(columns={0: 'Cluster 0', 1: 'Cluster 1', 2: 'Cluster 2', 3: 'Cluster 3', 4: 'Cluster 4'}) ### Item Features Matrix item_category = pd.DataFrame(cust_grouped.groupby(['Material', 'Prod Cat'])['Qty'].sum().unstack().fillna(0).reset_index().set_index('Material')) minmaxscaler = preprocessing.MinMaxScaler() item_category_scaled = minmaxscaler.fit_transform(item_category) item_category_scaled = pd.DataFrame(item_category_scaled) item_category_scaled.index = item_category.index item_category_scaled.columns = item_category.columns interaction_f = sparse.coo_matrix(interactions_scaled) user_f = sparse.coo_matrix(customer_features) item_f = sparse.coo_matrix(item_category_scaled) mask_size = len(interaction_f.data) np.random.choice(a=[False, True], size=mask_size, p=[.2, .8]) ## train, test data train_interactions, test_interactions = interaction_masking(interaction_f) user_features = user_f item_features = item_f # train ## 모델 파라미터 epochs = 100 alpha = 0.01 n_components = 10 verbose = True learning_rate = 0.01 n_sampled_items = int(item_features.shape[0] * .1) biased = False k_val = 100 model = TensorRec(n_components = n_components, user_repr_graph = DeepRepresentationGraph(), item_repr_graph = NormalizedLinearRepresentationGraph(), loss_graph = WMRBLossGraph(), biased=biased) model.fit(train_interactions, user_features, item_features, epochs=epochs, verbose=False, alpha=alpha, n_sampled_items=n_sampled_items, learning_rate=learning_rate) predicted_ranks = model.predict_rank(user_features=user_features, item_features=item_features) r_at_k_test = recall_at_k(predicted_ranks, test_interactions, k=80) r_at_k_train = recall_at_k(predicted_ranks, train_interactions, k=80) print("Recall at @k: Train: {:.2f} Test: {:.2f}".format(r_at_k_train.mean(), r_at_k_test.mean())) # produce the ranking into a readable table (dataframe it is) ranks_df = pd.DataFrame(predicted_ranks) ranks_df.columns = item_category_scaled.index ranks_df.index = customer_features.index ranks_df = ranks_df.T ranks_df.to_csv('./result/ranks_df.csv')
shape=(n_users, n_items)) # train collaborative filtering model epochs = 500 alpha = 0.00001 n_components = 10 verbose = True learning_rate = 0.01 n_sampled_items = int(n_items*0.01) fit_kwargs = {'epochs': epochs, 'alpha': alpha, 'verbose': verbose, 'learning_rate': learning_rate, 'n_sampled_items': n_sampled_items} cf_model = TensorRec(n_components=10, user_repr_graph=NormalizedLinearRepresentationGraph(), loss_graph=WMRBLossGraph()) cf_model.fit(user_features=user_features, item_features=item_features, interactions=train_interactions, **fit_kwargs) # calculate test ranks excluding training items predicted_ranks = cf_model.predict_rank(user_features=test_user_features, item_features=item_features) predicted_ranks[train.uid - 1, train.iid - 1] = n_items + 1 predicted_ranks = predicted_ranks.argsort(axis=1).argsort(axis=1) + 1 # evaluate precision and recall precision_results = precision_at_k(predicted_ranks, test_interactions, k=10) recall_results = recall_at_k(predicted_ranks, test_interactions, k=10) logging.info("Precision at 10: {}".format(np.mean(precision_results))) logging.info("Recall at 10: {}".format(np.mean(recall_results)))