def edit_account(request, name=None): if name and not has_role(request.user, ['staff', 'dev']): return redirect('edit_account') if name: user = McUser.objects.get(norm_name=normalize_name(name)).user else: user = request.user if request.method == 'POST': user_form = UserForm(request.POST, instance=user, user=user) if user_form.is_valid(): user_form.save() messages.add_message(request, messages.SUCCESS, 'Changes saved!') if name: return redirect('edit_other_account', name) return redirect('edit_account') else: messages.add_message(request, messages.WARNING, 'Passwords do not match.') user_form = UserForm(instance=user, user=user) else: user_form = UserForm(instance=user, user=user) context = { 'form': user_form, } if name: context['name'] = user.mcuser.get_full_name() return render(request, 'core/edit_account.html', context)
def audit(osmfile): ''' Performs the auditing operations on the given file. Returns a tuple (street_types, unnormalized_street_names), where street_types is a dictionary mapping unexpected street types to example street names with that type, and unnormalized_street_names is a set of street names that are not in normalized form. ''' street_types = defaultdict(set) unnormalized_street_names = set() for _, elem in logging_itr(ET.iterparse(osmfile)): if elem.tag == "node" or elem.tag == "way": street_name, street_type = get_street_name_and_type(elem) # Check for unexpected street types if street_type is not None and street_type not in expected: street_types[street_type].add(street_name) # Check for badly capitalized streets if street_name is not None and street_name != normalize_name(street_name): unnormalized_street_names.add(street_name) if elem.tag != 'tag': elem.clear() return street_types, unnormalized_street_names
def edit_honor(request, name): user_info = McUser.objects.get(norm_name=normalize_name(name)) if not user_info.user.id == request.user.id and not has_permission( request.user, 'edit_all_info'): return redirect('edit_honor', request.user.mcuser.norm_name) honor = Honor.objects.filter(user_id=user_info.id) if request.method == 'POST': honor_formset = HonorFormSet(request.POST, queryset=honor, initial=[{ 'user': user_info.id }]) if (honor_formset.is_valid()): honor_formset.save() messages.add_message( request, messages.SUCCESS, 'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[user_info.get_link_name()])) update_last_updated(user_info, request.user) return redirect('edit_honor', user_info.norm_name) else: honor_formset = HonorFormSet(queryset=honor, initial=[{ 'user': user_info.id }]) context = {'honor_formset': honor_formset, 'mcuser': user_info} return render(request, 'core/edit_honor.html', context)
def preprocess_mpd(mpd_data_path, exp_data_path): """ Since in our method we only use tracks and playlist names, from MPD we extract pID,trackID,title triplets and write them to another file to be used by our model! :param mpd_data_path: :param exp_data_path: :return: """ if not os.path.exists(exp_data_path): os.makedirs(exp_data_path) mpd_track_train_file = open( os.path.join(exp_data_path, settings.MPD_TRACK_TRAIN_FILE_NAME), 'w') filenames = os.listdir(mpd_data_path) for filename in sorted(filenames): if filename.startswith("mpd.slice.") and filename.endswith(".json"): fullpath = os.sep.join((mpd_data_path, filename)) f = open(fullpath) js = f.read() f.close() slice = json.loads(js) for playlist in slice['playlists']: pID = playlist['pid'] title = util.normalize_name(playlist['name']).encode('utf-8') for i, track in enumerate(playlist['tracks']): trackId = track['track_uri'] mpd_track_train_file.write( str(pID) + ',' + str(trackId) + ',' + title + '\n') mpd_track_train_file.flush() mpd_track_train_file.close()
def edit_info(request, name): user_info = McUser.objects.get(norm_name=normalize_name(name)) if not user_info.user.id == request.user.id and not has_permission(request.user, 'edit_all_info'): return redirect('edit_info', request.user.mcuser.get_link_name()) if request.method == 'POST': if has_role(request.user, 'staff'): form = McUserStaffForm(request.POST, request.FILES, instance=user_info, prefix='base') else: form = McUserForm(request.POST, request.FILES, instance=user_info, prefix='base') if (form.is_valid()): mcuser = form.save(commit=False) hidden_fields = [key.replace('checkbox_', '') for key in request.POST if key.startswith('checkbox_')] mcuser.hidden_fields = hidden_fields mcuser.save() messages.add_message( request, messages.SUCCESS, 'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[mcuser.get_link_name()])) update_last_updated(user_info, request.user) return redirect('edit_info', user_info.norm_name) else: if has_role(request.user, 'staff'): form = McUserStaffForm(instance=user_info, prefix='base') else: form = McUserForm(instance=user_info, prefix='base') context = { 'form': form, 'mcuser': user_info } template = 'core/edit_info.html' if has_role(user_info.user, 'staff'): template = 'core/edit_info_staff.html' return render(request, template, context)
def create_mapped_challenge_train(u_mapping_path, i_mapping_path): """ Assuming that by usings preprocess.py user and item mappings created, from challenge set we create train set for subprofile extraction and SPAD algorithm (For non-cold-start playlists)! There is no need to use the whole data set! Besides, write mapped piDs of the challenge set playlists to a file, because we will generate recommendations by using MF + SPAD for only those playlists. :param u_mapping_path: :param i_mapping_path: :return: """ CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH # Here write the list of unique tracks to a file maybe! MPD_DIR = settings.MPD_PATH EXP_DATA_DIR = os.path.join(MPD_DIR, settings.EXP_DIR_NAME) out_path = os.path.join(EXP_DATA_DIR, 'subprofile_train.csv') out_file = open(out_path, 'w') challenge_set_user_ids = set() # To write mapped challenge set playlist IDS to a file! challenge_set_user_ids_file = open( os.path.join(EXP_DATA_DIR, 'challenge_users.csv'), 'w') f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json")) js = f.read() challenge_set = json.loads(js) user_mapping = pickle.load(open(u_mapping_path, 'rb')) item_mapping = pickle.load(open(i_mapping_path, 'rb')) f.close() for playlist in challenge_set['playlists']: ntracks = playlist['num_samples'] if ntracks < 2: continue pID = playlist['pid'] mapped_pid = user_mapping[pID] if 'name' in playlist.keys(): name = util.normalize_name(playlist['name']).encode('utf-8') else: name = "" for track in playlist['tracks']: track_id = track['track_uri'] track_mapped_id = item_mapping[track_id] out_file.write( str(mapped_pid) + ',' + str(track_mapped_id) + ',' + name + '\n') challenge_set_user_ids.add(mapped_pid) out_file.flush() for uID in challenge_set_user_ids: challenge_set_user_ids_file.write(str(uID) + '\n') challenge_set_user_ids_file.flush() challenge_set_user_ids_file.close() out_file.close()
def update_name(name, mapping=STREET_MAPPING): ''' Cleans the given street name by replacing any suffix as specified by the given mapping, removing excess whitespace, and fixing capitalization. ''' name = normalize_name(name) street_base, street_type = split_street(name) if street_type is not None and street_type in mapping: name = street_base + mapping[street_type] return name
def profile(request, name): name = normalize_name(name) # TODO(joshcai): handle case where more than 1 user (aka users have same name) try: profile = McUser.objects.get(norm_name=name) except McUser.DoesNotExist: #TODO: check if URL with slash resolves then redirect to it if name in ('admin', 'calendar', 'documents', 'feedback', 'issues'): return redirect('%s/' % name) raise Http404('Page does not exist.') # 'user' is already passed in as default (the logged in user), # so use 'profile' as alias context = {'profile': profile, 'is_self': profile == request.user.mcuser} return render(request, 'core/profile.html', context)
def profile(request, name): name = normalize_name(name) # TODO(joshcai): handle case where more than 1 user (aka users have same name) try: profile = McUser.objects.get(norm_name=name) except McUser.DoesNotExist: #TODO: check if URL with slash resolves then redirect to it if name in ('admin', 'calendar', 'documents', 'feedback', 'issues'): return redirect('%s/' % name) raise Http404('Page does not exist.') # 'user' is already passed in as default (the logged in user), # so use 'profile' as alias context = { 'profile': profile, 'is_self': profile == request.user.mcuser } return render(request, 'core/profile.html', context)
def edit_edu(request, name): user_info = McUser.objects.get(norm_name=normalize_name(name)) if not user_info.user.id == request.user.id and not has_permission(request.user, 'edit_all_info'): return redirect('edit_edu', request.user.mcuser.norm_name) degrees = Degree.objects.filter(user_id=user_info.id) if request.method == 'POST': degrees_formset = DegreeFormSet(request.POST, queryset=degrees, initial=[{'user': user_info.id}]) if (degrees_formset.is_valid()): degrees_formset.save() messages.add_message( request, messages.SUCCESS, 'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[user_info.get_link_name()])) update_last_updated(user_info, request.user) return redirect('edit_edu', user_info.norm_name) else: degrees_formset = DegreeFormSet(queryset=degrees, initial=[{'user': user_info.id}]) context = { 'degrees_formset': degrees_formset, 'mcuser': user_info } return render(request, 'core/edit_edu.html', context)
def edit_info(request, name): user_info = McUser.objects.get(norm_name=normalize_name(name)) if not user_info.user.id == request.user.id and not has_permission( request.user, 'edit_all_info'): return redirect('edit_info', request.user.mcuser.get_link_name()) if request.method == 'POST': if has_role(request.user, 'staff'): form = McUserStaffForm(request.POST, request.FILES, instance=user_info, prefix='base') else: form = McUserForm(request.POST, request.FILES, instance=user_info, prefix='base') if (form.is_valid()): mcuser = form.save(commit=False) hidden_fields = [ key.replace('checkbox_', '') for key in request.POST if key.startswith('checkbox_') ] mcuser.hidden_fields = hidden_fields mcuser.save() messages.add_message( request, messages.SUCCESS, 'Changes saved! Click <a href="%s">here</a> to view profile.' % reverse('profile', args=[mcuser.get_link_name()])) update_last_updated(user_info, request.user) return redirect('edit_info', user_info.norm_name) else: if has_role(request.user, 'staff'): form = McUserStaffForm(instance=user_info, prefix='base') else: form = McUserForm(instance=user_info, prefix='base') context = {'form': form, 'mcuser': user_info} template = 'core/edit_info.html' if has_role(user_info.user, 'staff'): template = 'core/edit_info_staff.html' return render(request, template, context)
def preprocess_challenge_set(CHALLENGE_DATA_PATH, EXP_DATA_PATH): """ We will train our MF model by using known tracks in the challenge set so that for non-cold-start playlists in the challenge set we can learn latent factors (since we use challenge set as part of our train model this will be in creative track of the challenge!) we have to merge the known tracks in the challenge to the MPD train_track file! Besides we write the playlist with one song only to another file since our model for such playlist is different than others! For cold start scenarios see: title_popularity_recommendations.py => For playlists with title only. title_one_song_popularity_recommendations.py => For playlists with title + first track only. :param CHALLENGE_DATA_PATH: :param EXP_DATA_PATH: :return: """ challenge_data_file = open( os.path.join(EXP_DATA_PATH, settings.MPD_TRACK_TRAIN_FILE_NAME), 'a') cold_start_song_playlists_file = open( os.path.join(EXP_DATA_PATH, "one_song_playlists.csv"), 'w') f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json")) js = f.read() challenge_set = json.loads(js) f.close() for playlist in challenge_set['playlists']: if 'name' in playlist.keys(): name = util.normalize_name(playlist['name']).encode('utf-8') else: name = "" nKnownTracks = playlist['num_samples'] for track in playlist['tracks']: challenge_data_file.write( str(playlist['pid']) + ',' + str(track['track_uri']) + ',' + name + '\n') if nKnownTracks == 1: cold_start_song_playlists_file.write( str(playlist['pid']) + ',' + str(track['track_uri']) + ',' + name + '\n') cold_start_song_playlists_file.flush() challenge_data_file.flush() challenge_data_file.close() cold_start_song_playlists_file.close()
def main(): DATA_PATH = os.path.join(settings.MPD_PATH, settings.EXP_DIR_NAME) pop_file_path = os.path.join(DATA_PATH, 'pop') rec_file_path = os.path.join(DATA_PATH, 'pzt_100_50.0_SpeX_KNNJ_30_70_0.4') sub_file_path = os.path.join(DATA_PATH, 'sample_submission.csv') submission_file = open(sub_file_path, 'w') submission_file.write( "team_info,creative,teamrozik,[email protected]" + '\n') CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH title_pop_rec_path = os.path.join(DATA_PATH, 'title_popularity_recs.csv') title_one_song_pop_rec_path = os.path.join( DATA_PATH, 'title_one_song_popularity_recs.csv') print("Loading user and item mappings!") user_mapping_path = os.path.join(DATA_PATH, "user_mappings.tsv") user_mapping = pickle.load(open(user_mapping_path, 'rb')) item_mapping_path = os.path.join(DATA_PATH, "item_mappings.tsv") item_mapping = pickle.load(open(item_mapping_path, 'rb')) inverse_item_mapping = dict((v, k) for (k, v) in item_mapping.iteritems()) print("Finished loading user and item mappings!") pop_recs = load_pop_recommendations(pop_file_path, inverse_item_mapping) print("Finished Loading popularity recommendations!") # recs = load_recommendations(rec_file_path, inverse_item_mapping, inverse_user_mapping, c_users) all_recs = load_all_recs(rec_file_path) print("Finished Loading recommendations for non cold-start playlists!") # title_recs = generate_title_recommendations(title_mapping, all_recs, pop_recs, inverse_item_mapping) # print("Finished generating title recommendations!") title_pop_recs = load_title_popularity_recommendations(title_pop_rec_path) print( "Finished loading title populariy recommendations for cold start playlists!" ) title_one_song_pop_recs = load_title_popularity_recommendations( title_one_song_pop_rec_path) print( "Finished loading title populariy recommendations for one track only!") # Read challenge set! f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json")) js = f.read() challenge_set = json.loads(js) f.close() # First write the popularity recs for playlist in challenge_set['playlists']: pID = int(playlist['pid']) num_tracks = playlist['num_samples'] if num_tracks > 1: temp = get_recommendations_for_playlist(all_recs, user_mapping[pID], inverse_item_mapping) elif num_tracks == 1: trackID = [track['track_uri'] for track in playlist['tracks']][0] temp = get_one_song_cold_start_recs(str(pID), title_one_song_pop_recs, pop_recs, trackID) elif num_tracks == 0: name = util.normalize_name(playlist['name']).encode('utf-8') temp = get_total_cold_start_recs(name, title_pop_recs, pop_recs) rec_str = str(playlist['pid']) for rec in temp: rec_str += ',' + str(rec) rec_str += '\n' submission_file.write(rec_str) submission_file.flush() submission_file.close()
def save(self, *args, **kwargs): self.norm_name = normalize_name(self.get_full_name()) super(McUser, self).save(*args, **kwargs)
MPD_DATA_PATH = os.path.join(MPD_PATH, settings.MPD_DATA_DIR_NAME) EXP_DATA_PATH = os.path.join(MPD_PATH, settings.EXP_DIR_NAME) CHALLENGE_DATA_PATH = settings.CHALLENGE_DATA_PATH rec_file = open(os.path.join(EXP_DATA_PATH, 'title_popularity_recs.csv'), 'w') print("Loading challenge dataset!") cold_start_titles = set() f = open(os.path.join(CHALLENGE_DATA_PATH, "challenge_set.json")) js = f.read() challenge_set = json.loads(js) f.close() for playlist in challenge_set['playlists']: if playlist['num_samples'] < 1: name = util.normalize_name(playlist['name']).encode('utf-8') cold_start_titles.add(name) filenames = os.listdir(MPD_DATA_PATH) title_recs = dict() final_recs = defaultdict(list) for filename in sorted(filenames): print(filename) if filename.startswith("mpd.slice.") and filename.endswith(".json"): fullpath = os.sep.join((MPD_DATA_PATH, filename)) f = open(fullpath) js = f.read() f.close() slice = json.loads(js)
def main(): MPD_DATA_PATH = settings.MPD_PATH CONVERTED_DATA_PATH = os.path.join(MPD_DATA_PATH, settings.EXP_DIR_NAME) rec_file = open(os.path.join(CONVERTED_DATA_PATH, 'title_one_song_popularity_recs.csv'), 'w') cold_start_song_playlists_file_path = os.path.join(CONVERTED_DATA_PATH, "one_song_playlists.csv") cold_start_one_track_playlist = load_cold_start_one_track_playlists(cold_start_song_playlists_file_path) path = os.path.join(MPD_DATA_PATH, settings.MPD_DATA_DIR_NAME) filenames = os.listdir(path) title_recs = dict() final_recs = defaultdict(list) for filename in sorted(filenames): print(filename) if filename.startswith("mpd.slice.") and filename.endswith(".json"): fullpath = os.sep.join((path, filename)) f = open(fullpath) js = f.read() f.close() slice = json.loads(js) for playlist in slice['playlists']: name = util.normalize_name(playlist['name']).encode('utf-8') playlist_tracks = load_playlist_tracks(playlist) for csp in cold_start_one_track_playlist: val = 0 add = False cID, cTrackID, cName = csp if cName == name: val += 1 add = True if cTrackID in playlist_tracks: val += 1 add = True if add: for trackID in playlist_tracks: if trackID == cTrackID: continue if cID not in title_recs.keys(): temp_dict = dict() temp_dict[trackID] = val title_recs[cID] = temp_dict else: try: title_recs[cID][trackID] += val except: title_recs[cID][trackID] = val for pID, candidates in title_recs.iteritems(): temp = [] for track in candidates.keys(): temp.append((track, candidates[track])) temp.sort(key=operator.itemgetter(1), reverse=True) temp1 = temp[:500] mapped_recs = [val[0] for val in temp1] if len(mapped_recs) < 500: print("Not enough items to recommend!") print(len(temp1), len(mapped_recs)) final_recs[pID] = mapped_recs for pID, recs in final_recs.iteritems(): str_out = pID for rec in recs: str_out += ',' + str(rec) rec_file.write(str_out + '\n') rec_file.flush() rec_file.close()