def edit_account(request, name=None):
if name and not has_role(request.user, ['staff', 'dev']):
return redirect('edit_account')
if name:
user = McUser.objects.get(norm_name=normalize_name(name)).user
else:
user = request.user
if request.method == 'POST':
user_form = UserForm(request.POST, instance=user, user=user)
if user_form.is_valid():
user_form.save()
messages.add_message(request, messages.SUCCESS, 'Changes saved!')
if name:
return redirect('edit_other_account', name)
return redirect('edit_account')
else:
messages.add_message(request, messages.WARNING,
'Passwords do not match.')
user_form = UserForm(instance=user, user=user)
else:
user_form = UserForm(instance=user, user=user)
context = {
'form': user_form,
}
if name:
context['name'] = user.mcuser.get_full_name()
return render(request, 'core/edit_account.html', context)
def audit(osmfile):
'''
Performs the auditing operations on the given file. Returns a tuple
(street_types, unnormalized_street_names), where street_types is a
dictionary mapping unexpected street types to example street names with that
type, and unnormalized_street_names is a set of street names that are not in
normalized form.
'''
street_types = defaultdict(set)
unnormalized_street_names = set()
for _, elem in logging_itr(ET.iterparse(osmfile)):
if elem.tag == "node" or elem.tag == "way":
street_name, street_type = get_street_name_and_type(elem)
# Check for unexpected street types
if street_type is not None and street_type not in expected:
street_types[street_type].add(street_name)
# Check for badly capitalized streets
if street_name is not None and street_name != normalize_name(street_name):
unnormalized_street_names.add(street_name)
if elem.tag != 'tag':
elem.clear()
return street_types, unnormalized_street_names
def edit_honor(request, name):
user_info = McUser.objects.get(norm_name=normalize_name(name))
if not user_info.user.id == request.user.id and not has_permission(
request.user, 'edit_all_info'):
return redirect('edit_honor', request.user.mcuser.norm_name)
honor = Honor.objects.filter(user_id=user_info.id)
if request.method == 'POST':
honor_formset = HonorFormSet(request.POST,
queryset=honor,
initial=[{
'user': user_info.id
}])
if (honor_formset.is_valid()):
honor_formset.save()
messages.add_message(
request, messages.SUCCESS,
'Changes saved! Click <a href="%s">here</a> to view profile.' %
reverse('profile', args=[user_info.get_link_name()]))
update_last_updated(user_info, request.user)
return redirect('edit_honor', user_info.norm_name)
else:
honor_formset = HonorFormSet(queryset=honor,
initial=[{
'user': user_info.id
}])
context = {'honor_formset': honor_formset, 'mcuser': user_info}
return render(request, 'core/edit_honor.html', context)
def preprocess_mpd(mpd_data_path, exp_data_path):
"""
Since in our method we only use tracks and playlist names, from MPD we extract pID,trackID,title triplets and
write them to another file to be used by our model!
:param mpd_data_path:
:param exp_data_path:
:return:
"""
if not os.path.exists(exp_data_path):
os.makedirs(exp_data_path)
mpd_track_train_file = open(
os.path.join(exp_data_path, settings.MPD_TRACK_TRAIN_FILE_NAME), 'w')
filenames = os.listdir(mpd_data_path)
for filename in sorted(filenames):
if filename.startswith("mpd.slice.") and filename.endswith(".json"):
fullpath = os.sep.join((mpd_data_path, filename))
f = open(fullpath)
js = f.read()
f.close()
slice = json.loads(js)
for playlist in slice['playlists']:
pID = playlist['pid']
title = util.normalize_name(playlist['name']).encode('utf-8')
for i, track in enumerate(playlist['tracks']):
trackId = track['track_uri']
mpd_track_train_file.write(
str(pID) + ',' + str(trackId) + ',' + title + '\n')
mpd_track_train_file.flush()
mpd_track_train_file.close()
def edit_info(request, name):
user_info = McUser.objects.get(norm_name=normalize_name(name))
if not user_info.user.id == request.user.id and not has_permission(request.user, 'edit_all_info'):
return redirect('edit_info', request.user.mcuser.get_link_name())
if request.method == 'POST':
if has_role(request.user, 'staff'):
form = McUserStaffForm(request.POST, request.FILES, instance=user_info, prefix='base')
else:
form = McUserForm(request.POST, request.FILES, instance=user_info, prefix='base')
if (form.is_valid()):
mcuser = form.save(commit=False)
hidden_fields = [key.replace('checkbox_', '') for key in request.POST if key.startswith('checkbox_')]
mcuser.hidden_fields = hidden_fields
mcuser.save()
messages.add_message(
request, messages.SUCCESS,
'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[mcuser.get_link_name()]))
update_last_updated(user_info, request.user)
return redirect('edit_info', user_info.norm_name)
else:
if has_role(request.user, 'staff'):
form = McUserStaffForm(instance=user_info, prefix='base')
else:
form = McUserForm(instance=user_info, prefix='base')
context = {
'form': form,
'mcuser': user_info
}
template = 'core/edit_info.html'
if has_role(user_info.user, 'staff'):
template = 'core/edit_info_staff.html'
return render(request, template, context)
def audit(osmfile):
'''
Performs the auditing operations on the given file. Returns a tuple
(street_types, unnormalized_street_names), where street_types is a
dictionary mapping unexpected street types to example street names with that
type, and unnormalized_street_names is a set of street names that are not in
normalized form.
'''
street_types = defaultdict(set)
unnormalized_street_names = set()
for _, elem in logging_itr(ET.iterparse(osmfile)):
if elem.tag == "node" or elem.tag == "way":
street_name, street_type = get_street_name_and_type(elem)
# Check for unexpected street types
if street_type is not None and street_type not in expected:
street_types[street_type].add(street_name)
# Check for badly capitalized streets
if street_name is not None and street_name != normalize_name(street_name):
unnormalized_street_names.add(street_name)
if elem.tag != 'tag':
elem.clear()
return street_types, unnormalized_street_names
def edit_account(request, name=None):
if name and not has_role(request.user, ['staff', 'dev']):
return redirect('edit_account')
if name:
user = McUser.objects.get(norm_name=normalize_name(name)).user
else:
user = request.user
if request.method == 'POST':
user_form = UserForm(request.POST, instance=user, user=user)
if user_form.is_valid():
user_form.save()
messages.add_message(request, messages.SUCCESS, 'Changes saved!')
if name:
return redirect('edit_other_account', name)
return redirect('edit_account')
else:
messages.add_message(request, messages.WARNING, 'Passwords do not match.')
user_form = UserForm(instance=user, user=user)
else:
user_form = UserForm(instance=user, user=user)
context = {
'form': user_form,
}
if name:
context['name'] = user.mcuser.get_full_name()
return render(request, 'core/edit_account.html', context)
def create_mapped_challenge_train(u_mapping_path, i_mapping_path):
"""
Assuming that by usings preprocess.py user and item mappings created, from challenge set we create train set
for subprofile extraction and SPAD algorithm (For non-cold-start playlists)! There is no need to use the whole data set!
Besides, write mapped piDs of the challenge set playlists to a file, because we will generate recommendations
by using MF + SPAD for only those playlists.
:param u_mapping_path:
:param i_mapping_path:
:return:
"""
CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH
# Here write the list of unique tracks to a file maybe!
MPD_DIR = settings.MPD_PATH
EXP_DATA_DIR = os.path.join(MPD_DIR, settings.EXP_DIR_NAME)
out_path = os.path.join(EXP_DATA_DIR, 'subprofile_train.csv')
out_file = open(out_path, 'w')
challenge_set_user_ids = set()
# To write mapped challenge set playlist IDS to a file!
challenge_set_user_ids_file = open(
os.path.join(EXP_DATA_DIR, 'challenge_users.csv'), 'w')
f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json"))
js = f.read()
challenge_set = json.loads(js)
user_mapping = pickle.load(open(u_mapping_path, 'rb'))
item_mapping = pickle.load(open(i_mapping_path, 'rb'))
f.close()
for playlist in challenge_set['playlists']:
ntracks = playlist['num_samples']
if ntracks < 2:
continue
pID = playlist['pid']
mapped_pid = user_mapping[pID]
if 'name' in playlist.keys():
name = util.normalize_name(playlist['name']).encode('utf-8')
else:
name = ""
for track in playlist['tracks']:
track_id = track['track_uri']
track_mapped_id = item_mapping[track_id]
out_file.write(
str(mapped_pid) + ',' + str(track_mapped_id) + ',' + name +
'\n')
challenge_set_user_ids.add(mapped_pid)
out_file.flush()
for uID in challenge_set_user_ids:
challenge_set_user_ids_file.write(str(uID) + '\n')
challenge_set_user_ids_file.flush()
challenge_set_user_ids_file.close()
out_file.close()
def update_name(name, mapping=STREET_MAPPING):
'''
Cleans the given street name by replacing any suffix as specified by the
given mapping, removing excess whitespace, and fixing capitalization.
'''
name = normalize_name(name)
street_base, street_type = split_street(name)
if street_type is not None and street_type in mapping:
name = street_base + mapping[street_type]
return name
def profile(request, name):
name = normalize_name(name)
# TODO(joshcai): handle case where more than 1 user (aka users have same name)
try:
profile = McUser.objects.get(norm_name=name)
except McUser.DoesNotExist:
#TODO: check if URL with slash resolves then redirect to it
if name in ('admin', 'calendar', 'documents', 'feedback', 'issues'):
return redirect('%s/' % name)
raise Http404('Page does not exist.')
# 'user' is already passed in as default (the logged in user),
# so use 'profile' as alias
context = {'profile': profile, 'is_self': profile == request.user.mcuser}
return render(request, 'core/profile.html', context)
def profile(request, name):
name = normalize_name(name)
# TODO(joshcai): handle case where more than 1 user (aka users have same name)
try:
profile = McUser.objects.get(norm_name=name)
except McUser.DoesNotExist:
#TODO: check if URL with slash resolves then redirect to it
if name in ('admin', 'calendar', 'documents', 'feedback', 'issues'):
return redirect('%s/' % name)
raise Http404('Page does not exist.')
# 'user' is already passed in as default (the logged in user),
# so use 'profile' as alias
context = {
'profile': profile,
'is_self': profile == request.user.mcuser
}
return render(request, 'core/profile.html', context)
def edit_edu(request, name):
user_info = McUser.objects.get(norm_name=normalize_name(name))
if not user_info.user.id == request.user.id and not has_permission(request.user, 'edit_all_info'):
return redirect('edit_edu', request.user.mcuser.norm_name)
degrees = Degree.objects.filter(user_id=user_info.id)
if request.method == 'POST':
degrees_formset = DegreeFormSet(request.POST, queryset=degrees, initial=[{'user': user_info.id}])
if (degrees_formset.is_valid()):
degrees_formset.save()
messages.add_message(
request, messages.SUCCESS,
'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[user_info.get_link_name()]))
update_last_updated(user_info, request.user)
return redirect('edit_edu', user_info.norm_name)
else:
degrees_formset = DegreeFormSet(queryset=degrees, initial=[{'user': user_info.id}])
context = {
'degrees_formset': degrees_formset,
'mcuser': user_info
}
return render(request, 'core/edit_edu.html', context)
def edit_info(request, name):
user_info = McUser.objects.get(norm_name=normalize_name(name))
if not user_info.user.id == request.user.id and not has_permission(
request.user, 'edit_all_info'):
return redirect('edit_info', request.user.mcuser.get_link_name())
if request.method == 'POST':
if has_role(request.user, 'staff'):
form = McUserStaffForm(request.POST,
request.FILES,
instance=user_info,
prefix='base')
else:
form = McUserForm(request.POST,
request.FILES,
instance=user_info,
prefix='base')
if (form.is_valid()):
mcuser = form.save(commit=False)
hidden_fields = [
key.replace('checkbox_', '') for key in request.POST
if key.startswith('checkbox_')
]
mcuser.hidden_fields = hidden_fields
mcuser.save()
messages.add_message(
request, messages.SUCCESS,
'Changes saved! Click <a href="%s">here</a> to view profile.' %
reverse('profile', args=[mcuser.get_link_name()]))
update_last_updated(user_info, request.user)
return redirect('edit_info', user_info.norm_name)
else:
if has_role(request.user, 'staff'):
form = McUserStaffForm(instance=user_info, prefix='base')
else:
form = McUserForm(instance=user_info, prefix='base')
context = {'form': form, 'mcuser': user_info}
template = 'core/edit_info.html'
if has_role(user_info.user, 'staff'):
template = 'core/edit_info_staff.html'
return render(request, template, context)
def preprocess_challenge_set(CHALLENGE_DATA_PATH, EXP_DATA_PATH):
"""
We will train our MF model by using known tracks in the challenge set so that for non-cold-start playlists in the
challenge set we can learn latent factors (since we use challenge set as part of our train model this will be in
creative track of the challenge!) we have to merge the known tracks in the challenge to the MPD train_track file!
Besides we write the playlist with one song only to another file since our model for such playlist is different than others!
For cold start scenarios see:
title_popularity_recommendations.py => For playlists with title only.
title_one_song_popularity_recommendations.py => For playlists with title + first track only.
:param CHALLENGE_DATA_PATH:
:param EXP_DATA_PATH:
:return:
"""
challenge_data_file = open(
os.path.join(EXP_DATA_PATH, settings.MPD_TRACK_TRAIN_FILE_NAME), 'a')
cold_start_song_playlists_file = open(
os.path.join(EXP_DATA_PATH, "one_song_playlists.csv"), 'w')
f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json"))
js = f.read()
challenge_set = json.loads(js)
f.close()
for playlist in challenge_set['playlists']:
if 'name' in playlist.keys():
name = util.normalize_name(playlist['name']).encode('utf-8')
else:
name = ""
nKnownTracks = playlist['num_samples']
for track in playlist['tracks']:
challenge_data_file.write(
str(playlist['pid']) + ',' + str(track['track_uri']) + ',' +
name + '\n')
if nKnownTracks == 1:
cold_start_song_playlists_file.write(
str(playlist['pid']) + ',' + str(track['track_uri']) +
',' + name + '\n')
cold_start_song_playlists_file.flush()
challenge_data_file.flush()
challenge_data_file.close()
cold_start_song_playlists_file.close()
def main():
DATA_PATH = os.path.join(settings.MPD_PATH, settings.EXP_DIR_NAME)
pop_file_path = os.path.join(DATA_PATH, 'pop')
rec_file_path = os.path.join(DATA_PATH, 'pzt_100_50.0_SpeX_KNNJ_30_70_0.4')
sub_file_path = os.path.join(DATA_PATH, 'sample_submission.csv')
submission_file = open(sub_file_path, 'w')
submission_file.write(
"team_info,creative,teamrozik,[email protected]" + '\n')
CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH
title_pop_rec_path = os.path.join(DATA_PATH, 'title_popularity_recs.csv')
title_one_song_pop_rec_path = os.path.join(
DATA_PATH, 'title_one_song_popularity_recs.csv')
print("Loading user and item mappings!")
user_mapping_path = os.path.join(DATA_PATH, "user_mappings.tsv")
user_mapping = pickle.load(open(user_mapping_path, 'rb'))
item_mapping_path = os.path.join(DATA_PATH, "item_mappings.tsv")
item_mapping = pickle.load(open(item_mapping_path, 'rb'))
inverse_item_mapping = dict((v, k) for (k, v) in item_mapping.iteritems())
print("Finished loading user and item mappings!")
pop_recs = load_pop_recommendations(pop_file_path, inverse_item_mapping)
print("Finished Loading popularity recommendations!")
# recs = load_recommendations(rec_file_path, inverse_item_mapping, inverse_user_mapping, c_users)
all_recs = load_all_recs(rec_file_path)
print("Finished Loading recommendations for non cold-start playlists!")
# title_recs = generate_title_recommendations(title_mapping, all_recs, pop_recs, inverse_item_mapping)
# print("Finished generating title recommendations!")
title_pop_recs = load_title_popularity_recommendations(title_pop_rec_path)
print(
"Finished loading title populariy recommendations for cold start playlists!"
)
title_one_song_pop_recs = load_title_popularity_recommendations(
title_one_song_pop_rec_path)
print(
"Finished loading title populariy recommendations for one track only!")
# Read challenge set!
f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json"))
js = f.read()
challenge_set = json.loads(js)
f.close()
# First write the popularity recs
for playlist in challenge_set['playlists']:
pID = int(playlist['pid'])
num_tracks = playlist['num_samples']
if num_tracks > 1:
temp = get_recommendations_for_playlist(all_recs,
user_mapping[pID],
inverse_item_mapping)
elif num_tracks == 1:
trackID = [track['track_uri'] for track in playlist['tracks']][0]
temp = get_one_song_cold_start_recs(str(pID),
title_one_song_pop_recs,
pop_recs, trackID)
elif num_tracks == 0:
name = util.normalize_name(playlist['name']).encode('utf-8')
temp = get_total_cold_start_recs(name, title_pop_recs, pop_recs)
rec_str = str(playlist['pid'])
for rec in temp:
rec_str += ',' + str(rec)
rec_str += '\n'
submission_file.write(rec_str)
submission_file.flush()
submission_file.close()
def save(self, *args, **kwargs):
self.norm_name = normalize_name(self.get_full_name())
super(McUser, self).save(*args, **kwargs)
MPD_DATA_PATH = os.path.join(MPD_PATH, settings.MPD_DATA_DIR_NAME)
EXP_DATA_PATH = os.path.join(MPD_PATH, settings.EXP_DIR_NAME)
CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH
rec_file = open(os.path.join(EXP_DATA_PATH, 'title_popularity_recs.csv'), 'w')
print("Loading challenge dataset!")
cold_start_titles = set()
f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json"))
js = f.read()
challenge_set = json.loads(js)
f.close()
for playlist in challenge_set['playlists']:
if playlist['num_samples'] < 1:
name = util.normalize_name(playlist['name']).encode('utf-8')
cold_start_titles.add(name)
filenames = os.listdir(MPD_DATA_PATH)
title_recs = dict()
final_recs = defaultdict(list)
for filename in sorted(filenames):
print(filename)
if filename.startswith("mpd.slice.") and filename.endswith(".json"):
fullpath = os.sep.join((MPD_DATA_PATH, filename))
f = open(fullpath)
js = f.read()
f.close()
slice = json.loads(js)
def main():
MPD_DATA_PATH = settings.MPD_PATH
CONVERTED_DATA_PATH = os.path.join(MPD_DATA_PATH, settings.EXP_DIR_NAME)
rec_file = open(os.path.join(CONVERTED_DATA_PATH, 'title_one_song_popularity_recs.csv'), 'w')
cold_start_song_playlists_file_path = os.path.join(CONVERTED_DATA_PATH, "one_song_playlists.csv")
cold_start_one_track_playlist = load_cold_start_one_track_playlists(cold_start_song_playlists_file_path)
path = os.path.join(MPD_DATA_PATH, settings.MPD_DATA_DIR_NAME)
filenames = os.listdir(path)
title_recs = dict()
final_recs = defaultdict(list)
for filename in sorted(filenames):
print(filename)
if filename.startswith("mpd.slice.") and filename.endswith(".json"):
fullpath = os.sep.join((path, filename))
f = open(fullpath)
js = f.read()
f.close()
slice = json.loads(js)
for playlist in slice['playlists']:
name = util.normalize_name(playlist['name']).encode('utf-8')
playlist_tracks = load_playlist_tracks(playlist)
for csp in cold_start_one_track_playlist:
val = 0
add = False
cID, cTrackID, cName = csp
if cName == name:
val += 1
add = True
if cTrackID in playlist_tracks:
val += 1
add = True
if add:
for trackID in playlist_tracks:
if trackID == cTrackID:
continue
if cID not in title_recs.keys():
temp_dict = dict()
temp_dict[trackID] = val
title_recs[cID] = temp_dict
else:
try:
title_recs[cID][trackID] += val
except:
title_recs[cID][trackID] = val
for pID, candidates in title_recs.iteritems():
temp = []
for track in candidates.keys():
temp.append((track, candidates[track]))
temp.sort(key=operator.itemgetter(1), reverse=True)
temp1 = temp[:500]
mapped_recs = [val[0] for val in temp1]
if len(mapped_recs) < 500:
print("Not enough items to recommend!")
print(len(temp1), len(mapped_recs))
final_recs[pID] = mapped_recs
for pID, recs in final_recs.iteritems():
str_out = pID
for rec in recs:
str_out += ',' + str(rec)
rec_file.write(str_out + '\n')
rec_file.flush()
rec_file.close()
def save(self, *args, **kwargs): self.norm_name = normalize_name(self.get_full_name()) super(McUser, self).save(*args, **kwargs)
