def getGoodUsers(): """ get list of users ids that have visited greater or equal than <MIN_EVENTS> number of events """ print "Getting list of users that have enough visits..." # fill dictionary of visits users_dict = {} history_file = open(history_file_name, 'r') # skip 3 lines of comments for i in range(3): history_file.readline() for line in history_file: user_id = misc_functions.getMeta(line, 0) item_id = misc_functions.getMeta(line, 1) if user_id in users_dict: users_dict[user_id].append(item_id) else: users_dict[user_id] = [] # filter out users with less than <MIN_EVENTS> visits good_users_list = [] for user_id in users_dict: if len(users_dict[user_id]) >= MIN_EVENTS: good_users_list.append(user_id) print "Filtered users with enough visits: ", len(good_users_list) return good_users_list
def getGoodUsers(): """ get list of users ids that have visited greater or equal than <MIN_EVENTS> number of events """ print "Getting list of users that have enough visits..." # fill dictionary of visits users_dict = {} history_file = open(history_file_name, 'r') # skip 3 lines of comments for i in range(3): history_file.readline() for line in history_file: user_id = misc_functions.getMeta(line, 0) item_id = misc_functions.getMeta(line, 1) if user_id in users_dict: users_dict[user_id].append(item_id) else: users_dict[user_id] = [] # filter out users with less than <MIN_EVENTS> visits good_users_list = [] for user_id in users_dict: if len(users_dict[user_id]) >= MIN_EVENTS: good_users_list.append(user_id) print "Filtered users with enough visits: ", len(good_users_list) return good_users_list
def rewriteUsersData(users_IDs_map, good_users_list): """ creates new file with users meta; in fact just replaces old users IDs with new IDs """ print "Writing new users data..." users_file = open(users_file_name, 'r') good_users_lines = [] for user_line in users_file: old_user_id = misc_functions.getMeta(user_line, 0) if old_user_id in good_users_list: rest_of_line = user_line[user_line.find('\t'):] good_users_lines.append(str(users_IDs_map[old_user_id]) + rest_of_line) users_file.close() # sort users lines by new user id sorted_users_lines = misc_functions.sortMetaListByMeta(good_users_lines, 0) # write new file with users data new_users_file = open(users_file_name + ".preprocessed", 'w') for user_line in sorted_users_lines: new_users_file.write(user_line) new_users_file.close() print "Users data with new IDs written."
def rewriteUsersData(users_IDs_map, good_users_list): """ creates new file with users meta; in fact just replaces old users IDs with new IDs """ print "Writing new users data..." users_file = open(users_file_name, 'r') good_users_lines = [] for user_line in users_file: old_user_id = misc_functions.getMeta(user_line, 0) if old_user_id in good_users_list: rest_of_line = user_line[user_line.find('\t'):] good_users_lines.append( str(users_IDs_map[old_user_id]) + rest_of_line) users_file.close() # sort users lines by new user id sorted_users_lines = misc_functions.sortMetaListByMeta(good_users_lines, 0) # write new file with users data new_users_file = open(users_file_name + ".preprocessed", 'w') for user_line in sorted_users_lines: new_users_file.write(user_line) new_users_file.close() print "Users data with new IDs written."
def createNewEventsIDs(): """ sorts events by time; creates dictionary with key - old event ID, meaning - new event ID """ print "Creating new IDs for events..." print "Sorting events by time..." time_sorted_meta_list = misc_functions.sortMetaListByTime( global_meta_list, TIME_META_POSITION) # start counting from 1 new_event_id = 1 events_IDs_map = {} for event_line in time_sorted_meta_list: old_event_id = misc_functions.getMeta(event_line, 0) events_IDs_map[old_event_id] = new_event_id new_event_id += 1 print "New IDs for events created." return [events_IDs_map, time_sorted_meta_list]
def createItem_x_MetaMatrix_global(): """ Function computes item_X_meta matrix for all the seminars (that shows current meta), and saves this matrix as "items-metas_global.mtx" in "data/well_done" directory. """ print "STARTED computing global Item_x_Meta matrix" seminars_meta_file = open("../data/well_done/meta", 'r') item_X_meta_matrix = numpy.zeros((ITEMS_NUMB, MAX_METAS_NUMB), dtype = int) for seminar_id in range(ITEMS_NUMB): seminar_string = seminars_meta_file.readline() #for meta in METAS_TO_USE: for meta in range(18): item_X_meta_matrix[seminar_id][meta] = misc_functions.getMeta(seminar_string, meta) seminars_meta_file.close() csr_matrix_to_write = scipy.sparse.csr_matrix(item_X_meta_matrix) scipy.io.mmio.mmwrite("../data/well_done/items-metas_global", csr_matrix_to_write) print "SUCCESS! \n ITEMS x METAS global matrix - prepared!"
def createVisitsMatrix( users_IDs_map, events_IDs_map, \ good_users_list, good_events_list): """ creates new matrix of visits; result matrix contains only events with meta data and users with more than <MIN_EVENTS> visits each; """ print "Creating new visits matrix..." old_history_file = open(history_file_name, 'r') # skip 3 lines of header for i in range(3): old_history_file.readline() new_history_lines = [] for line in old_history_file: old_user_id = misc_functions.getMeta(line, 0) old_event_id = misc_functions.getMeta(line, 1) if (old_user_id in good_users_list) and (old_event_id in good_events_list): ltw = str(users_IDs_map[old_user_id]) + '\t' + str( events_IDs_map[old_event_id]) + '\t' + '1' + '\n' new_history_lines.append(ltw) old_history_file.close() new_history_file = open(history_file_name + ".preprocessed", 'w') # write header for new history file new_history_file.write( "%%MatrixMarket matrix coordinate integer general\n") new_history_file.write("%% Created by CTHULHU\n") new_history_file.write(str(len(good_users_list)) + '\t' + \ str(len(good_events_list)) + '\t' + \ str(len(new_history_lines)) + '\n' ) for line in new_history_lines: new_history_file.write(line) new_history_file.close() print "New visits matrix created."
def createVisitsMatrix( users_IDs_map, events_IDs_map, \ good_users_list, good_events_list): """ creates new matrix of visits; result matrix contains only events with meta data and users with more than <MIN_EVENTS> visits each; """ print "Creating new visits matrix..." old_history_file = open(history_file_name, 'r') # skip 3 lines of header for i in range(3): old_history_file.readline() new_history_lines = [] for line in old_history_file: old_user_id = misc_functions.getMeta(line, 0) old_event_id = misc_functions.getMeta(line, 1) if (old_user_id in good_users_list) and (old_event_id in good_events_list): ltw = str(users_IDs_map[old_user_id]) + '\t' + str(events_IDs_map[old_event_id]) + '\t' + '1' + '\n' new_history_lines.append(ltw) old_history_file.close() new_history_file = open(history_file_name + ".preprocessed", 'w') # write header for new history file new_history_file.write("%%MatrixMarket matrix coordinate integer general\n") new_history_file.write("%% Created by CTHULHU\n") new_history_file.write(str(len(good_users_list)) + '\t' + \ str(len(good_events_list)) + '\t' + \ str(len(new_history_lines)) + '\n' ) for line in new_history_lines: new_history_file.write(line) new_history_file.close() print "New visits matrix created."
def rewriteEventsData(events_IDs_map, time_sorted_meta_list): """ creates new file with events meta; in fact just replaces old events IDs with new IDs; takes into account that seminars has been time sorted already """ print "Writing new events data..." new_events_file = open(events_file_name + ".preprocessed", 'w') for event_line in time_sorted_meta_list: old_event_id = misc_functions.getMeta(event_line, 0) rest_of_line = event_line[event_line.find('\t'):] new_events_file.write(str(events_IDs_map[old_event_id]) + rest_of_line) new_events_file.close() print "Events data with new IDs and time-sorted events written."
def rewriteEventsData(events_IDs_map, time_sorted_meta_list): """ creates new file with events meta; in fact just replaces old events IDs with new IDs; takes into account that seminars has been time sorted already """ print "Writing new events data..." new_events_file = open(events_file_name + ".preprocessed", 'w') for event_line in time_sorted_meta_list: old_event_id = misc_functions.getMeta(event_line, 0) rest_of_line = event_line[event_line.find('\t'):] new_events_file.write(str(events_IDs_map[old_event_id]) + rest_of_line) new_events_file.close() print "Events data with new IDs and time-sorted events written."
def getGoodEvents(): """ get list of events ids that have meta data; IMPORTANT: fills list of meta data for events - global var <global_meta_list> """ print "Getting list of events ids that have meta data..." events_file = open(events_file_name, 'r') events_with_meta_list = [] for line in events_file: global_meta_list.append(line) event_id = misc_functions.getMeta(line, 0) if event_id not in events_with_meta_list: events_with_meta_list.append(event_id) events_file.close() print "Filtered events with meta data: ", len(events_with_meta_list) return events_with_meta_list
def getGoodEvents(): """ get list of events ids that have meta data; IMPORTANT: fills list of meta data for events - global var <global_meta_list> """ print "Getting list of events ids that have meta data..." events_file = open(events_file_name, 'r') events_with_meta_list = [] for line in events_file: global_meta_list.append(line) event_id = misc_functions.getMeta(line, 0) if event_id not in events_with_meta_list: events_with_meta_list.append(event_id) events_file.close() print "Filtered events with meta data: ", len(events_with_meta_list) return events_with_meta_list
def createNewEventsIDs(): """ sorts events by time; creates dictionary with key - old event ID, meaning - new event ID """ print "Creating new IDs for events..." print "Sorting events by time..." time_sorted_meta_list = misc_functions.sortMetaListByTime(global_meta_list, TIME_META_POSITION) # start counting from 1 new_event_id = 1 events_IDs_map = {} for event_line in time_sorted_meta_list: old_event_id = misc_functions.getMeta(event_line, 0) events_IDs_map[old_event_id] = new_event_id new_event_id += 1 print "New IDs for events created." return [events_IDs_map, time_sorted_meta_list]
def computeMetaMatrix(meta_list, meta_id_position): """ Internal function. Function computes user_X_meta matrix for current <meta_id_position> (that shows current meta), and saves this matrix as "users-<meta>.mtx in directory for current case. """ #print "meta_id_position = ", meta_id_position # creating new matrix meta_matrix = numpy.zeros((USERS_NUMB, 0), dtype = int) # reading local history matrix history_matrix = scipy.io.mmio.mmread("history.mtx").tocsr() # some routine before main loop cur_meta_items = [] """ id-- here????? """ #cur_meta_id = 1 cur_meta_id = 0 # MAIN LOOP for line in meta_list: # seminar with unknown meta is ignored if (misc_functions.getMeta(line, meta_id_position) == -1): continue """ !!!! DAFUQ!!!! """ line_semin_id = getMeta(line, 0) # considering everywhere semin_id_position == 0 #line_meta_id = int(line.split('\t')[meta_id_position]) line_meta_id = getMeta(line, meta_id_position) #print line #print line_semin_id #print line_meta_id #step() # PRICE HEURISTIC if meta_id_position == 8: line_meta_id = priceToPriceCat(line_meta_id) # TIME HEURISTIC if meta_id_position == 5: line_meta_id = dayTime(line_meta_id) #print line_semin_id #step() # if new meta_id detected if line_meta_id != cur_meta_id: print line_meta_id new_meta_col = numpy.zeros((USERS_NUMB, 1), dtype = int) for cur_item in cur_meta_items: cur_item_col = (history_matrix[ : , cur_item]).toarray() new_meta_col = new_meta_col + cur_item_col if cur_meta_items != []: meta_matrix = numpy.hstack((meta_matrix, new_meta_col)) # stacking empty columns of meta for not visited seminars while(line_meta_id != cur_meta_id + 1): meta_matrix = numpy.hstack((meta_matrix, numpy.zeros((USERS_NUMB, 1), dtype = int))) cur_meta_id += 1 # clean list if new meta_id begins #print "meta_id = ", cur_meta_id, "; visiters = ", len(cur_meta_items) cur_meta_items = [] """ !!! """ cur_meta_id += 1 if cur_meta_id != line_meta_id: raise Exception("mismatch cur_meta_id") """ !!! """ cur_meta_items.append(line_semin_id) # stacking last column new_meta_col = numpy.zeros((USERS_NUMB, 1), dtype = int) for cur_item in cur_meta_items: cur_item_col = (history_matrix[ : , cur_item]).toarray() new_meta_col = new_meta_col + cur_item_col if cur_meta_items != []: meta_matrix = numpy.hstack((meta_matrix, new_meta_col)) meta_matrix_csr = scipy.sparse.csr_matrix(meta_matrix) # writing new <meta_matrix> to file now = datetime.datetime.now() now_string = now.strftime("%Y-%m-%d %H:%M") meta_matrix_file_name = "users-" + METAS_TO_USE[meta_id_position] scipy.io.mmio.mmwrite(meta_matrix_file_name, meta_matrix_csr, now_string, 'integer')
if ".." not in sys.path: sys.path.insert(0, "..") import misc_functions # file with events meta descriptions dataset_dir = "../../data/datasets" dataset_name = "Timepad/raw" events_file_name = "events.preprocessed" events_file = open(dataset_dir + "/" + dataset_name + "/" + events_file_name, 'r') # file to save new meta lines meta_numb = 8 #meta_name = "headers" meta_file = open(dataset_dir + "/" + dataset_name + "/times", 'w') for line in events_file: ltw = "" #print line #id = misc_functions.getMeta(line, 0) #ltw = ltw + str(id) + '\t' meta = str(misc_functions.getMeta(line, meta_numb)) ltw += meta meta_file.write(ltw + '\n') events_file.close() meta_file.close()
if ".." not in sys.path: sys.path.insert(0, "..") import misc_functions # file with events meta descriptions dataset_dir = "../../data/datasets" dataset_name = "Timepad/raw" events_file_name = "events.preprocessed" events_file = open(dataset_dir + "/" + dataset_name + "/" + events_file_name, 'r') # file to save new meta lines meta_numb = 8 #meta_name = "headers" meta_file = open(dataset_dir + "/" + dataset_name + "/times", 'w') for line in events_file: ltw = "" #print line #id = misc_functions.getMeta(line, 0) #ltw = ltw + str(id) + '\t' meta = str( misc_functions.getMeta(line, meta_numb) ) ltw += meta meta_file.write(ltw + '\n') events_file.close() meta_file.close()
def estimateNDCGp( test_matrix, prediction_matrix, clusters_list, coords, results_file_name ): """ Function estimates accuracy of prediction in each case of cross-validation - it gets results of prediction from "prediction.mtx", test matrix from "test.mtx", and get info about clusters from "test_clusters" file. Function uses normalized DCGp metrics - it counts error for each cluster and get average error between clusters per user and then average for all users in current cross-validation case. """ # file stuff: remove after check - we get matrices from call above #prediction_matrix = scipy.io.mmio.mmread(prediction_file_name).tocsr() #test_matrix = scipy.io.mmio.mmread("test.mtx").tocsr() """ # == DEBUG PRINT ===================================================== for user_cluster in clusters_list: for line in user_cluster: print line print "Press any key to continue:" sys.stdin.read(1) # == \DEBUG PRINT ===================================================== """ local_average_nDCGp = float(0.0) local_average_p = float(0.0) #user_ctr = 0 for user_cluster in clusters_list: #misc_functions.step() #print "user = "******"user_cluster = ", user_cluster #print "user_cluster[0] = ", user_cluster[0] user = int(((user_cluster[0]).split("\t"))[1]) #print "user = "******"user_prediction = ", user_prediction #print "user_visits = ", user_visits user_average_nDCGp = float(0.0) user_average_p = float(0.0) for byte in user_cluster[1 : ] : #misc_functions.step() byte_visits = user_visits[misc_functions.getMeta(byte, 0) : misc_functions.getMeta(byte, 2) + 1] byte_predictions = user_prediction[misc_functions.getMeta(byte, 0) : misc_functions.getMeta(byte, 2) + 1] #print "byte_visits = ", byte_visits #print "byte_predictions = ", byte_predictions # get indices of columns by decreasing of prediction value indices = numpy.lexsort(keys = (-byte_predictions, -byte_predictions)) ideal_indices = numpy.lexsort(keys = (byte_visits, byte_visits)) #print "indices = ", indices #print "ideal_indices = ", ideal_indices # sort vector of visits and predictions with help of indices sorted_predictions = byte_predictions.take(indices, axis = 0) sorted_visits = byte_visits.take(indices, axis = 0) #sorted_predictions = numpy.sort(sorted_predictions) #sorted_visits = #print "sorted_visits = ", sorted_visits #print "sorted_predictions = ", sorted_predictions if len(sorted_predictions) != len(sorted_visits): raise Exception("visits and prediction clusters have different size") #print "sorted_visits[] = ", sorted_visits """ TO BE NORMALIZED """ nDCGp = float(0.0) p = len(sorted_predictions) for i in range(p): if sorted_visits[i] == 1: nDCGp = float(math.log(2, i + 2)) break """ for i in range(1, p + 1): #print "i" nDCGp += float(sorted_visits[i - 1]) / float(math.log(i + 1, 2)) """ """ / TO BE NORMALIZED """ #if (nDCGp > 1.0): #nDCGp = 1.0 #print "nDCGp = ", nDCGp #step() #if (nDCGp < 0.0): #raise Exception("Incorrect nDCGp") if (nDCGp < 0.0) or (nDCGp > 1.0): print "nDCGp = ", nDCGp raise Exception("Incorrect nDCGp") #print "nDCGp = ", nDCGp user_average_nDCGp += nDCGp user_average_p += p if (len(user_cluster) != 1): # WHY -1?? user_average_nDCGp /= (len(user_cluster) - 1) # WHY -1?? user_average_p /= (len(user_cluster) - 1) local_average_nDCGp += user_average_nDCGp local_average_p += user_average_p local_average_nDCGp /= (len(clusters_list)) local_average_p /= (len(clusters_list)) if local_average_nDCGp != 0: local_average_position = math.pow(2, 1.0 / local_average_nDCGp) else: local_average_position = "FAR UNKNOWN" print "nDCGp for case = ", local_average_nDCGp print "average p = ", local_average_p print "average position", local_average_position results_file = open(results_file_name + ".nDCGp", 'w') results_file.write(str(local_average_nDCGp)) results_file.close() return [local_average_nDCGp, local_average_p, local_average_position]
def estimateNDCGp(test_matrix, prediction_matrix, clusters_list, coords, results_file_name): """ Function estimates accuracy of prediction in each case of cross-validation - it gets results of prediction from "prediction.mtx", test matrix from "test.mtx", and get info about clusters from "test_clusters" file. Function uses normalized DCGp metrics - it counts error for each cluster and get average error between clusters per user and then average for all users in current cross-validation case. """ # file stuff: remove after check - we get matrices from call above #prediction_matrix = scipy.io.mmio.mmread(prediction_file_name).tocsr() #test_matrix = scipy.io.mmio.mmread("test.mtx").tocsr() """ # == DEBUG PRINT ===================================================== for user_cluster in clusters_list: for line in user_cluster: print line print "Press any key to continue:" sys.stdin.read(1) # == \DEBUG PRINT ===================================================== """ local_average_nDCGp = float(0.0) local_average_p = float(0.0) #user_ctr = 0 for user_cluster in clusters_list: #misc_functions.step() #print "user = "******"user_cluster = ", user_cluster #print "user_cluster[0] = ", user_cluster[0] user = int(((user_cluster[0]).split("\t"))[1]) #print "user = "******"user_prediction = ", user_prediction #print "user_visits = ", user_visits user_average_nDCGp = float(0.0) user_average_p = float(0.0) for byte in user_cluster[1:]: #misc_functions.step() byte_visits = user_visits[misc_functions.getMeta(byte, 0): misc_functions.getMeta(byte, 2) + 1] byte_predictions = user_prediction[misc_functions.getMeta( byte, 0):misc_functions.getMeta(byte, 2) + 1] #print "byte_visits = ", byte_visits #print "byte_predictions = ", byte_predictions # get indices of columns by decreasing of prediction value indices = numpy.lexsort(keys=(-byte_predictions, -byte_predictions)) ideal_indices = numpy.lexsort(keys=(byte_visits, byte_visits)) #print "indices = ", indices #print "ideal_indices = ", ideal_indices # sort vector of visits and predictions with help of indices sorted_predictions = byte_predictions.take(indices, axis=0) sorted_visits = byte_visits.take(indices, axis=0) #sorted_predictions = numpy.sort(sorted_predictions) #sorted_visits = #print "sorted_visits = ", sorted_visits #print "sorted_predictions = ", sorted_predictions if len(sorted_predictions) != len(sorted_visits): raise Exception( "visits and prediction clusters have different size") #print "sorted_visits[] = ", sorted_visits """ TO BE NORMALIZED """ nDCGp = float(0.0) p = len(sorted_predictions) for i in range(p): if sorted_visits[i] == 1: nDCGp = float(math.log(2, i + 2)) break """ for i in range(1, p + 1): #print "i" nDCGp += float(sorted_visits[i - 1]) / float(math.log(i + 1, 2)) """ """ / TO BE NORMALIZED """ #if (nDCGp > 1.0): #nDCGp = 1.0 #print "nDCGp = ", nDCGp #step() #if (nDCGp < 0.0): #raise Exception("Incorrect nDCGp") if (nDCGp < 0.0) or (nDCGp > 1.0): print "nDCGp = ", nDCGp raise Exception("Incorrect nDCGp") #print "nDCGp = ", nDCGp user_average_nDCGp += nDCGp user_average_p += p if (len(user_cluster) != 1): # WHY -1?? user_average_nDCGp /= (len(user_cluster) - 1) # WHY -1?? user_average_p /= (len(user_cluster) - 1) local_average_nDCGp += user_average_nDCGp local_average_p += user_average_p local_average_nDCGp /= (len(clusters_list)) local_average_p /= (len(clusters_list)) if local_average_nDCGp != 0: local_average_position = math.pow(2, 1.0 / local_average_nDCGp) else: local_average_position = "FAR UNKNOWN" print "nDCGp for case = ", local_average_nDCGp print "average p = ", local_average_p print "average position", local_average_position results_file = open(results_file_name + ".nDCGp", 'w') results_file.write(str(local_average_nDCGp)) results_file.close() return [local_average_nDCGp, local_average_p, local_average_position]