class Server():
def __init__(self, conf, data):
self.conf = conf
self.data = data
self.fed_clients = self.conf.fed_clients
self.syft_clients = self.conf.syft_clients
self.init()
def init(self):
print("-> initial server")
self.helper = Helper(self.conf)
self.eval = Evaluation(self.conf, self.data)
self.model = Model(num_features=self.conf.num_features, num_classes=self.conf.num_classes).to(self.conf.device)
def run(self):
# training process
for t in range(self.conf.num_round):
start = time.time()
# random select client
curr_client_ids = np.random.choice(self.conf.num_clients, self.conf.num_per_round, replace=False)
print(f"\n\n-> t={t} selected_client={curr_client_ids}")
# client update
spdz_weights = {}
for uid in curr_client_ids:
spdz_weights[uid] = self.fed_clients[uid].update(self.model.state_dict())
if not self.conf.fed_horizontal["encrypt_weight"]:
print("-> client send params to server without shared_encrypt")
new_weights = self.aggregation(spdz_weights)
self.model.copy_params(new_weights)
stop = time.time()
print("-> end round:{:d} using:{:.2f}s".format(t, float(stop-start)))
# evluation
self.eval.eval_test(self.model)
def aggregation(self, spdz_weights):
if len(spdz_weights) == 1:
return spdz_weights[0]
if self.conf.fed_aggregate == "avg":
return self.helper.weights_avg(self.model.state_dict(), spdz_weights,
fix_precision=self.conf.fed_horizontal["encrypt_weight"])
def __init__(self, URM_train, mode="validation"):
self.URM_train = URM_train
# Init single recommenders
self.top_pop = HybridUserCBFRegionalTopPop(URM_train)
self.SSLIM = MultiThreadSSLIM_ElasticNet(URM_train)
self.UCF = UserCollaborativeFilter(URM_train)
self.user_cbf = UserCBF(URM_train)
# Get the cold users list
self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train)
# Fit the single recommenders, this saves time in case of changing just the relative weights
self.top_pop.fit()
self.SSLIM.fit(**SSLIM_parameters)
self.UCF.fit(**UCF_parameters)
self.user_cbf.fit(**user_cbf_parameters)
def fit(self,
topK=410,
shrink=0,
normalize=True,
similarity="cosine",
bm_25_norm=False):
self.topK = topK
self.shrink = shrink
self.normalize = normalize
self.similarity = similarity
if bm_25_norm:
self.URM_train = Helper().bm25_normalization(self.URM_train)
self.W_sparse = self.compute_similarity_matrix(self.URM_train,
self.shrink, self.topK,
self.normalize,
self.similarity)
def evaluate_fgan_loss(inet, fnet, dataloader, nactors):
print('Evaluate Fusion Network')
inet.eval()
fnet.eval()
top1 = AverageMeter()
top5 = AverageMeter()
for batch_idx, (inputs, targets) in enumerate(dataloader):
z_missed, z_missed_g, target_missed, _ = batch_process(
inet, fnet, inputs, targets, nactors)
out_missed_g = inet.module.classifier(z_missed_g)
prec1, prec5 = Helper.accuracy(out_missed_g,
target_missed,
topk=(1, 5))
top1.update(prec1.item(), target_missed.size(0))
top5.update(prec5.item(), target_missed.size(0))
del z_missed, z_missed_g, target_missed, _
return top1.avg, top5.avg
def search_comments(self,
q,
page=0,
page_size=DEFAULT_PAGE_SIZE,
startdate=None,
enddate=None):
session = self.db_helper.Session()
try:
news = session.query(News).filter(
News.news_id == self.news_id).first()
query = session.query(Comments).filter(
Comments.news_id == self.news_id).filter(
Comments.comment.contains(q))
t_query = session.query(func.count('*').label('total')).filter(
Comments.news_id == self.news_id).filter(
Comments.comment.contains(q))
if startdate and startdate != '':
query = query.filter(
Comments.comment_time >= Helper.get_date(startdate))
t_query = t_query.filter(
Comments.comment_time >= Helper.get_date(startdate))
if enddate and enddate != '':
query = query.filter(
Comments.comment_time <= Helper.get_date(enddate))
t_query = t_query.filter(
Comments.comment_time <= Helper.get_date(enddate))
query = query.order_by(Comments.comment_time.desc())
results = self.db_helper.query(query,
page=page,
page_size=page_size)
total_comments = int(t_query[0][0])
pages = int(total_comments /
page_size) if total_comments % page_size == 0 else int(
total_comments / page_size) + 1
return {
'dates': {
'start': (Helper.get_date(startdate).strftime('%Y-%m-%d')
if startdate and startdate != '' else ''),
'end': (Helper.get_date(enddate).strftime('%Y-%m-%d')
if enddate and enddate != '' else '')
},
'pages': pages,
'comments': [result.to_dict() for result in results],
'news': news.to_dict()
}
except Exception as ex:
self.logger.error("Exception occurred when searching comments. ",
ex)
return {'pages': 0, 'comments': []}
finally:
session.close()
def __parse_comments(self, response):
for sel in response.xpath("//div[@id='comments']/ul/li"):
try:
item = response.meta['item']
item['comment_id'] = int(
sel.xpath("@data-cid").extract_first().strip())
item['comment'] = sel.xpath(
"div[@class='comment']/p[1]/span[1]//text()"
).extract_first().strip()
item['comment_time'] = Helper.parse_comment_time(
sel.xpath(
"div[@class='comment']/h3[1]/span[2]/span[2]/text()").
extract_first().strip())
yield item
except Exception as ex:
self.logger.error(
f"Exception occurred when parsing comment with response {response}",
ex)
yield None
def objective(params):
print(params)
total_loss = 0
for k in range(4):
URM_train, URM_test, validation_data, test_data = Helper().get_kfold_data(4)[k]
booster = XGBooster(URM_train, validation_data, HybridElasticNetICFUCF)
booster.URM_test = URM_test
booster.fit(train_parameters=deepcopy(params))
loss, _ = Evaluator(test_mode=True).evaluate_recommender_kfold(booster, test_data, sequential=True)
total_loss += loss
total_loss /= 4
print("Map@10 k-fold score:", total_loss)
return -total_loss
def run(recommender_class,
fit_parameters,
init_params=None,
at=10,
submission_path="data/recommendation_matrix_to_submit.csv"):
# Helper contains methods to convert URM in CSR
if init_params is None:
init_params = {}
URM_all = Helper().URM_csr
recommender = recommender_class(URM_all, **init_params)
recommender.fit(**fit_parameters)
RunRecommender.write_submission(recommender,
submission_path=submission_path,
at=at)
def init(self, *weight):
print("-> initial server")
self.helper = Helper(self.conf)
# creat homomorphic encryption key pair
# TODO
# init weight
if not len(weight):
weight = [None, None]
# setup model
for uid, party in self.party.items():
self.model[uid] = Model(party.num_features,
party.num_output,
weight=weight[uid]).to(self.conf.device)
self.optimizer[uid] = optim.SGD(self.model[uid].parameters(),
lr=self.conf.learning_rate,
momentum=self.conf.momentum)
def __init__(self, URM_train, mode="dataset"):
self.URM_train = URM_train
# Init single recommenders
self.item_cbf = ItemCBF(URM_train)
self.item_cf = ItemCollaborativeFilter(URM_train)
self.top_pop = HybridUserCBFRegionalTopPop(URM_train)
self.SLIM = MultiThreadSLIM_ElasticNet(URM_train)
self.rp3 = RP3betaRecommender(URM_train)
# Get the cold users list
self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train)
# Fit the single recommenders, this saves time in case of changing just the relative weights
self.item_cbf.fit(**item_cbf_parameters)
self.item_cf.fit(**item_cf_parameters)
self.rp3.fit(**rp3_parameters)
self.top_pop.fit()
self.SLIM.fit(**SLIM_parameters)
def __init__(
self,
URM_train,
mode="validation",
):
self.URM_train = URM_train
# Init single recommenders
self.user_cf = UserCollaborativeFilter(URM_train)
self.item_cf = ItemCollaborativeFilter(URM_train)
self.top_pop = TopPopRecommender(URM_train)
self.als = AlternatingLeastSquare(URM_train)
# Get the cold users list
self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train)
# Fit the single recommenders, this saves time in case of changing just the relative weights
self.user_cf.fit(**user_cf_parameters)
self.item_cf.fit(**item_cf_parameters)
self.top_pop.fit()
self.als.fit(**als_parameters)
def get_all_userfeedbacks(self):
"""
Get all users'feedbacks from table user's feedback
"""
# Read User Feedback from csv
user_feedback = UserFeedback(**self.config)
### Read all feedbacks from table user's feedback
criteria = {}
df_userfb = user_feedback.get_user_feedback(criteria)
df_userfb = df_userfb.dropna()
# Filter by Feedback_App name; Feedback_App= YOUTUBE_GENERAL
try:
## Remove from column RTT: the values Blank space, NaN, Zero, #NULL!, +Inf
df_userfb[self.config['COL']['RTT']] = df_userfb[
self.config['COL']['RTT']].apply(lambda x: self.__to_number(x))
## Recording the inconsistent instances index
dropIx = df_userfb[df_userfb[self.config['COL']['RTT']] ==
-1].index
## Dropping these instances from the dataset:
df_userfb.drop(dropIx, inplace=True)
df_userfb.reset_index(inplace=True)
# Rename columns to match the columns name in model
df_userfb = Helper.rename_df_columnname(df_userfb,
self.config['COL'])
# Save for testing purpose
filename = self.config['OUT_DIR'] + 'all_ufb_except_unknown.csv'
df_userfb.to_csv(filename)
except IOError as error:
print(error)
return df_userfb
def fit(self,
top_pop_weight=0.02139131367609725,
topK=765,
shrink=6,
normalize=True,
bm_25_norm=False,
similarity="jaccard",
suppress_interactions=False,
asymmetric_alpha=0.5):
if bm_25_norm:
self.URM_train = Helper().bm25_normalization(self.URM_train)
self.toppop = RegionalTopPopRecommender(self.URM_train)
self.user_cbf = UserCBF(self.URM_train)
self.user_cbf.fit(topK=topK,
shrink=shrink,
normalize=normalize,
similarity=similarity,
suppress_interactions=suppress_interactions,
asymmetric_alpha=asymmetric_alpha)
self.toppop.fit()
self.toppop_weight = top_pop_weight
def import_todo_list(request):
if request.method == 'POST':
result = Helper.message()
try:
file = request.FILES.get('csv_file', None)
try:
Helper.validate_file_extension(file, '.csv')
except ValidationError:
result[
'message'] = 'File type is not valid.Please select a csv file.'
decoded_file = file.read().decode('ISO-8859-1').replace(
'', '').splitlines()
file_content = list(
csv.reader(decoded_file, delimiter=';', quotechar='"'))
header = file_content.pop(0)
todo_index = Helper.column_index('Todo', header)
status_index = Helper.column_index('Status', header)
authenticated_user_id = Helper.get_session(request).user_id
if todo_index is not -1 and status_index is not -1:
for r in file_content:
try:
Todo.objects.create(
user_id=authenticated_user_id,
text=r[todo_index].strip(),
is_completed=False
if 'Not Completed' in r[status_index] else True)
except Exception as ex:
print(ex)
result = Helper.message_success(text='Records are created.')
else:
result['message'] = 'File format is not valid.'
except Exception as ex:
print(ex)
result['message'] = str(ex)
return JsonResponse(result)
return render(request, 'pages/todo/import_form.html', {})
def perform_evaluation(recommender):
"""Takes an already fitted recommender and evaluates on test data.
If test_mode is false writes the submission"""
print("Performing evaluation on test set...")
MAP_final = 0.0
evaluator, helper = Evaluator(), Helper()
URM_train, eval_data = helper.URM_train_validation, helper.validation_data
recommender.fit(URM_train)
for user in tqdm(eval_data.keys()):
recommended_items = recommender.recommend(int(user),
exclude_seen=True)
relevant_item = eval_data[int(user)]
MAP_final += evaluator.MAP(recommended_items, relevant_item)
MAP_final /= len(eval_data.keys())
print("MAP-10 score:", MAP_final)
MAP_final *= 0.665
print("MAP-10 public approx score:", MAP_final)
return MAP_final
def __init__(
self,
URM_train,
mode="validation",
):
self.URM_train = URM_train
# Init single recommenders
self.user_cf = UserCollaborativeFilter(URM_train)
self.item_cf = ItemCollaborativeFilter(URM_train)
self.top_pop = TopPopRecommender(URM_train)
self.SLIM = MultiThreadSLIM_ElasticNet(URM_train)
# self.user_based_cbf = UserBasedCBF(URM_train)
# self.item_based_cbf = ItemBasedCBF(URM_train)
# Get the cold users list
self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train)
# Fit the single recommenders, this saves time in case of changing just the relative weights
self.user_cf.fit(**user_cf_parameters)
self.item_cf.fit(**item_cf_parameters)
self.top_pop.fit()
self.SLIM.fit(**SLIM_parameters)
def plot_reconstruction(model_name,
sample_path,
inet,
fnet,
dataloader,
nactors,
concat_input=False):
print('Evaluate fusion network: ' + model_name)
inet.eval()
fnet.eval()
corrects = [0, 0, 0]
criterionL1 = torch.nn.L1Loss(reduction='mean')
iters = iter(dataloader)
(inputs, targets) = iters.next()
inputs = Variable(inputs).cuda()
targets = Variable(targets).cuda()
N, C, H, W = inputs.shape
M = N // nactors
N = M * nactors
inputs = inputs[:N, ...].view(M, nactors, C, H, W)
targets = targets[:N, ...].view(M, nactors)
# fusion inputs
x_g = inputs.view(M, nactors * C, H, W)
x_fused_g = rescale(fnet(x_g))
# Z
_, z, _ = inet(inputs.view(M * nactors, C, H, W))
z = z.view((M, nactors, z.shape[1], z.shape[2], z.shape[3]))
target_missed = targets[:, 0]
z_missed = z[:, 0, ...]
z_rest = z[:, 1:, ...].sum(dim=1)
z_fused = z.mean(dim=1)
_, z_fused_g, _ = inet(x_fused_g)
z_missed_g = nactors * z_fused_g - z_rest
# classification
out_missed_g = inet.module.classifier(z_missed_g)
out_missed = inet.module.classifier(z_missed)
# evaluation
_, y = torch.max(out_missed.data, 1)
_, y_g = torch.max(out_missed_g.data, 1)
corrects[0] = y.eq(target_missed.data).sum().cpu().item()
corrects[1] = y_g.eq(target_missed.data).sum().cpu().item()
corrects[2] = y_g.eq(y.data).sum().cpu().item()
# inverse
x_fused_g = inet.module.inverse(z_fused_g)
x_fused = inet.module.inverse(z_fused)
x_missed = inet.module.inverse(z_missed)
x_missed_g = inet.module.inverse(z_missed_g)
# visualize
Helper.save_images(x_fused_g, sample_path, model_name, 'x_fused_g', 0)
Helper.save_images(x_fused, sample_path, model_name, 'x_fused', 0)
Helper.save_images(x_missed_g, sample_path, model_name, 'x_missed_g',
0)
Helper.save_images(x_missed, sample_path, model_name, 'x_missed', 0)
print("L1: {:.4f}".format(
criterionL1(torch.tanh(x_fused_g), torch.tanh(x_fused))))
print(np.asarray(corrects) / M)
from IPython import embed
embed()
# HybridUCFICFRecommender = HybridUCFICFRecommender(Helper().URM_train_test)
recommender_class = HybridSSLIMICFUCFRP3Beta
N_KFOLD = 10
kfold = False
parallel_fit = True
if kfold > 0:
MAP_final = 0
recommender_list = []
print("Getting Kfold splits...\n")
kfold_data = Helper().get_kfold_data(N_KFOLD)
print("Done!\n")
for i in range(N_KFOLD):
print("Fitting recommender", i+1, "...\n")
recommender_list.append(HybridSSLIMICFUCFRP3Beta(kfold_data[i][0], multithreaded=True))
print("\nCompleted!\n")
else:
hybrid = HybridSSLIMICFUCFRP3Beta(Helper().URM_train_validation)
# Step 1 : defining the objective function
def objective(params):
print("\n############## New iteration ##############\n", params)
if kfold:
loss = - RunRecommender.evaluate_hybrid_weights_validation_kfold(recommender_list, params, kfold=N_KFOLD, parallelize_evaluation=kfold, parallel_fit=False)
recommender_class = Hybrid
recommenders = [MultiThreadSLIM_ElasticNet, RP3betaRecommender, ItemCBF, AlternatingLeastSquare]
N_KFOLD = 6
kfold = True
if kfold > 0:
MAP_final = 0
recommender_list = []
print("Getting Kfold splits...\n")
kfold_data = Helper().get_kfold_data(N_KFOLD)
print("Done!\n")
for i in range(N_KFOLD):
print("Fitting recommender", i+1, "...\n")
recommender_list.append(recommender_class(kfold_data[i][0], recommenders))
print("\nCompleted!\n")
else:
hybrid = recommender_class(Helper().URM_train_validation, recommenders)
# Step 1 : defining the objective function
def objective(params):
print("\n############## New iteration ##############\n", params)
params = {"weights": params}
if kfold:
class Evaluator:
def __init__(self, split_size=0.8):
self.helper = Helper()
self.URM_data = self.helper.URM_data # Matrix containing full original training data
self.URM_test = pd.DataFrame(columns=["playlist_id", "track_id"
]) # Matrix containing test data
self.URM_train = None # Matrix containing reduced training data, at the beginning it is equal to the test data (then it will be removed)
self.target_playlists_test = None # New list of playlists for which MAP will be computed
self.split_size = split_size # Percentage of training set that will be kept for training purposes (1-split_size is the size of the test set)
def split_data_randomly(self):
# Instantiate an array containing the playlists to be put in new target_playlist_file
playlists_list = np.asarray(list(self.URM_data.playlist_id))
URM_csr = self.helper.convert_URM_data_to_csr()
# Will contain target playlist ids of the test file
self.target_playlists_test = np.empty(shape=playlists_list.shape)
# Group by the URM just to have faster search
grouped_by_playlist_URM = self.URM_data.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
# Number of tuples of the test set, used for the upper bound over the counter
test_dimensions = len(playlists_list) * (1 - self.split_size)
for count in tqdm(range(int(test_dimensions))):
tracks_less_than_10 = True
# Don't consider items with less than 10 elements (would not be good for testing since Kaggle evaluates with MAP10)
while tracks_less_than_10:
# Choose a random index between 0 and the length of URM's number of playlists
random_index_to_be_put_in_test = randint(
0,
len(playlists_list) - 1)
candidate_playlist = playlists_list[
random_index_to_be_put_in_test]
# If there the playlist has fewer than 10 songs, we have to discard it
if len(URM_csr[candidate_playlist].indices
) >= NUMBER_OF_TRACKS_TEST_FILE:
# Append random playlist to playlist test set
self.target_playlists_test[count] = playlists_list[
random_index_to_be_put_in_test]
# np array containing all the tracks contained in a selected playlist
track_list_10 = np.asarray(
list(grouped_by_playlist_URM[candidate_playlist]))
# Keep just 10 tracks for each playlist
playlist_tracks_list = track_list_10[:
NUMBER_OF_TRACKS_TEST_FILE]
# Create a tuple to be appended in URM_test matrix and append it
URM_test_tuple = pd.DataFrame({
"playlist_id":
[int(self.target_playlists_test[count])],
"track_id": [str(playlist_tracks_list)]
})
self.URM_test = self.URM_test.append(URM_test_tuple,
ignore_index=True)
# Exit while loop
tracks_less_than_10 = False
# Format data of test_data.csv correctly and save it
self.URM_test["track_id"] = self.URM_test["track_id"].str.strip(' []')
self.URM_test["track_id"] = self.URM_test["track_id"].replace(
'\s+', ' ', regex=True)
self.URM_test.to_csv(os.path.join(ROOT_PROJECT_PATH,
"data/test_data.csv"),
index=False)
print("URM_test created, now creating URM_train")
self.target_playlists_test = np.asarray(list(
self.URM_test.playlist_id))
self.URM_train = self.URM_data
for playlist in tqdm(self.target_playlists_test):
#print(self.URM_data.loc[self.URM_data["playlist_id"] != playlist])
self.URM_train = self.URM_train.loc[~(
self.URM_train["playlist_id"] == playlist)]
self.URM_train.to_csv(os.path.join(ROOT_PROJECT_PATH,
"data/train_data.csv"),
index=False)
target_playlists_new_csv_file = open(
os.path.join(ROOT_PROJECT_PATH, "data/target_playlists_test.csv"),
"w")
target_playlists_new_csv_file.write("playlist_id\n")
for playlist in tqdm(self.target_playlists_test):
target_playlists_new_csv_file.write(str(int(playlist)) + "\n")
def __init__(self, URM):
self.URM_train = URM
self.helper = Helper()
class AssetCBF:
def __init__(self, URM):
self.URM_train = URM
self.helper = Helper()
def compute_similarity_cbf(self,
ICM,
top_k,
shrink,
normalize=True,
similarity="cosine"):
# Compute similarities for weighted features recommender
similarity_object = Compute_Similarity_Python(ICM.T,
shrink=shrink,
topK=top_k,
normalize=normalize,
similarity=similarity)
w_sparse = similarity_object.compute_similarity()
return w_sparse
def fit(self, topK, shrink):
self.topK = topK
self.shrink = shrink
# Load ICMs from helper
self.ICM_asset = self.helper.bm25_normalization(
self.helper.load_icm_asset())
# Computing SMs
self.SM_asset = self.compute_similarity_cbf(self.ICM_asset,
top_k=self.topK,
shrink=self.shrink)
def compute_scores(self, user_id):
users_list_train = self.URM_train[user_id]
scores_asset = users_list_train.dot(self.SM_asset).toarray().ravel()
return scores_asset
def recommend(self, user_id, at=10, exclude_seen=True):
# Compute scores of the recommendation
scores = self.compute_scores(user_id)
# Filter to exclude already seen items
if exclude_seen:
scores = self.filter_seen(user_id, scores)
recommended_items = np.argsort(scores)
recommended_items = np.flip(recommended_items, axis=0)
return recommended_items[:at]
def filter_seen(self, user_id, scores):
start_pos = self.URM_train.indptr[user_id]
end_pos = self.URM_train.indptr[user_id + 1]
user_profile = self.URM_train.indices[start_pos:end_pos]
scores[user_profile] = -np.inf
return scores
import hyperopt as hp
from hyperopt import Trials, fmin, STATUS_OK
from Hybrid_ALS_SLIMElastic import HybridSSLIMUCF
from utils.run import RunRecommender
from utils.helper import Helper
hybrid = HybridSSLIMUCF(Helper().URM_train_validation, mode="validation")
### Step 1 : defining the objective function
def objective(params):
print("Current parameters:")
print(params)
loss = - RunRecommender.evaluate_hybrid_weights_validation(hybrid, params)
return loss
als_space = {
"SLIM_weight": hp.hp.uniform('SLIM_weight', 0.7, 1.0),
}
if __name__ == '__main__':
### step 3 : storing the results of every iteration
bayes_trials = Trials()
MAX_EVALS = 50
# Optimize
best = fmin(fn=objective, space=als_space, algo=hp.tpe.suggest,
max_evals=MAX_EVALS, trials=bayes_trials, verbose=True)
def defaults(request):
return {
'is_admin': Helper.is_admin(request),
'user': Helper.get_session(request),
}
def init(self):
print("-> initial server")
self.helper = Helper(self.conf)
self.eval = Evaluation(self.conf, self.data)
self.model = Model(num_features=self.conf.num_features, num_classes=self.conf.num_classes).to(self.conf.device)
class CBFRecomender:
def __init__(self,
knn_artist=25,
knn_album=45,
shrink_artist=0,
shrink_album=8,
weight_artist=0.15):
self.knn_artist = knn_artist
self.knn_album = knn_album
self.shrink_artist = shrink_artist
self.shrink_album = shrink_album
self.weight_artist = weight_artist
self.helper = Helper()
def compute_similarity_cbf(self,
ICM,
top_k,
shrink,
normalize=True,
similarity="cosine"):
#Compute similarities for weighted features recommender
similarity_object = Compute_Similarity_Python(ICM.T,
shrink=shrink,
topK=top_k,
normalize=normalize,
similarity=similarity)
w_sparse = similarity_object.compute_similarity()
return w_sparse
def fit(self, URM):
# URM Loading
self.URM = URM
# Load ICMs from helper
self.ICM_artist = self.helper.load_icm_artist()
self.ICM_artist = self.helper.tfidf_normalization(self.ICM_artist)
self.ICM_artist = self.helper.sklearn_normalization(self.ICM_artist)
self.ICM_album = self.helper.load_icm_album()
self.ICM_album = self.helper.bm25_normalization(self.ICM_album)
# Computing SMs
self.SM_artist = self.compute_similarity_cbf(self.ICM_artist,
top_k=self.knn_artist,
shrink=self.shrink_artist)
self.SM_album = self.compute_similarity_cbf(self.ICM_album,
top_k=self.knn_album,
shrink=self.shrink_album)
def compute_scores(self, playlist_id):
tracks_list_train = self.URM[playlist_id]
scores_artist = tracks_list_train.dot(self.SM_artist).toarray().ravel()
scores_album = tracks_list_train.dot(self.SM_album).toarray().ravel()
weight_album = 1 - self.weight_artist
scores = (scores_artist * self.weight_artist) + (scores_album *
weight_album)
return scores
def recommend(self, playlist_id, at=10, exclude_seen=True):
# Compute scores of the recommendation
scores = self.compute_scores(playlist_id)
# Filter to exclude already seen items
if exclude_seen:
scores = self.filter_seen(playlist_id, scores)
recommended_items = np.argsort(scores)
recommended_items = np.flip(recommended_items, axis=0)
return recommended_items[:at]
def filter_seen(self, playlist_id, scores):
start_pos = self.URM.indptr[playlist_id]
end_pos = self.URM.indptr[playlist_id + 1]
user_profile = self.URM.indices[start_pos:end_pos]
scores[user_profile] = -np.inf
return scores
import hyperopt as hp
from hyperopt import Trials, fmin, space_eval, STATUS_OK
from SLIM_BPR.Cython.SLIM_BPR_Cython import SLIM_BPR_Cython
from evaluation.Evaluator import Evaluator
from utils.run import RunRecommender
import numpy as np
from utils.helper import Helper
helper = Helper()
N_KFOLD = 10
MAX_EVALS = 100
evaluator = Evaluator()
### Step 1 : defining the objective function
def objective(params):
params["topK"] = int(params["topK"])
params["batch_size"] = int(params["batch_size"])
params["random_seed"] = 1234
params["epochs"] = 30
print("############ Current parameters ############")
print(params)
loss = -RunRecommender.evaluate_on_validation_set(
SLIM_BPR_Cython,
params,
Kfold=N_KFOLD,
parallelize_evaluation=False,
user_group="warm")
def eval_inv(model,
testloader,
sample_path,
model_name,
num_epochs,
nactors,
debug=False):
model_name = "{}_epoch_{}".format(model_name, num_epochs)
print('Evaluate Invertibility on ', model_name)
model.eval()
# loss
criterion = torch.nn.MSELoss(reduction='sum')
corrects = [0, 0, 0] # correct-x, correct-x-hat, match
total = 0
in_shapes = -1
for batch_idx, (inputs, targets) in enumerate(testloader):
# print(batch_idx)
targets = Variable(targets).cuda()
inputs = Variable(inputs).cuda()
batch_size, C, H, W = inputs.shape
total += batch_size
out, z_input, _ = model(inputs)
x = inputs.clone().unsqueeze(0).expand(
nactors - 1, batch_size, C, H, W).contiguous().view(
(nactors - 1) * batch_size, C, H, W)
x = x[torch.randperm(x.shape[0]), :]
_, z_c, _ = model(x)
z_c = z_c.view(nactors - 1, batch_size, z_input.shape[1],
z_input.shape[2], z_input.shape[3]).sum(dim=0)
z_fused = (z_input + z_c) / nactors
x_fused = model.module.inverse(z_fused)
_, z_fused_hat, _ = model(x_fused)
z_hat = z_fused_hat * nactors - z_c
# invert
x_inv = model.module.inverse(z_input)
x_inv_hat = model.module.inverse(z_hat)
# classification
out_hat = model.module.classifier(z_hat)
_, y = torch.max(out.data, 1)
_, y_hat = torch.max(out_hat.data, 1)
corrects[0] += y.eq(targets.data).sum().cpu().item()
corrects[1] += y_hat.eq(targets.data).sum().cpu().item()
corrects[2] += y_hat.eq(y.data).sum().cpu().item()
# save samples
if batch_idx == 0:
Helper.save_images(x_inv, sample_path, model_name, 'inv',
batch_idx)
Helper.save_images(x_inv_hat, sample_path, model_name,
'inv_hat', batch_idx)
Helper.save_images(x_fused, sample_path, model_name, 'fused',
batch_idx)
del out, out_hat, z_c, z_fused, z_hat, z_fused_hat, inputs, x_inv, x_inv_hat, targets, y, y_hat
# print('\t {} images of {} pixels'.format(total, in_shapes))
corrects = 100 * np.asarray(corrects) / total
print('\t Correctly classified: X {:.4f} X_hat {:.4f} Match {:.4f}'.
format(corrects[0], corrects[1], corrects[2]))
return corrects[0]
from hyperopt import hp, tpe, fmin, Trials
from hyperopt import STATUS_OK, STATUS_FAIL
import pickle
import os
import traceback
__author__ = "Guillaume Chevalier"
__copyright__ = "Copyright 2017, Guillaume Chevalier"
__license__ = "MIT License"
__notice__ = (
"Some further edits by Guillaume Chevalier are made on "
"behalf of Vooban Inc. and belongs to Vooban Inc. ")
# See: https://github.com/Vooban/Hyperopt-Keras-CNN-CIFAR-100/blob/master/LICENSE"
h = Helper('topomaps_RT_100/train/combined/', 'topomaps_RT_100/test/combined/', 'results/RT_100')
space = {
# This loguniform scale will multiply the learning rate, so as to make
# it vary exponentially, in a multiplicative fashion rather than in
# a linear fashion, to handle his exponentialy varying nature:
'lr_rate_mult': hp.loguniform('lr_rate_mult', -0.5, 0.5),
# L2 weight decay:
'l2_weight_reg_mult': hp.loguniform('l2_weight_reg_mult', -1.3, 1.3),
# Batch size fed for each gradient update
'batch_size': hp.quniform('batch_size', 40, 128, 10),
# Number of EPOCHS
'epochs': hp.quniform('epochs', 10, 120, 10),
# Choice of optimizer:
'optimizer': hp.choice('optimizer', ['Adam', 'Nadam', 'RMSprop']),
# Uniform distribution in finding appropriate dropout values, conv layers
from utils.provider import FusionDataset
from configs import args
############ Settings ##############
TRAIN_FRACTION = 0.8
cudnn.benchmark = True
torch.cuda.manual_seed(args.seed)
device = torch.device("cuda:0")
args.fnet_name = "fnet_{}_{}_{}_{}".format(args.fname, args.dataset, args.iname, args.nactors)
args.inet_name = "inet_{}_{}".format(args.dataset, args.iname)
checkpoint_dir = os.path.join(args.save_dir, 'checkpoints')
args.inet_save_dir = os.path.join(checkpoint_dir, args.inet_name)
args.fnet_save_dir = os.path.join(checkpoint_dir, args.fnet_name)
Helper.try_make_dir(args.save_dir)
Helper.try_make_dir(checkpoint_dir)
Helper.try_make_dir(args.fnet_save_dir)
Helper.try_make_dir(args.inet_save_dir)
if args.dataset == 'cifar10':
args.input_nc = 3
args.output_nc = 3
in_shape = (3, 32, 32)
else:
args.input_nc = 1
args.output_nc = 1
in_shape = (1, 32, 32)
fusion_data_dir = os.path.join(args.data_dir, "fusion/{}_{}_{}".format(args.dataset, args.iname, args.nactors))
train_path = os.path.join(fusion_data_dir, 'train.npy')
