def load_CSV_into_SparseBuilder(filePath, header=False, separator="::"): matrixBuilder = IncrementalSparseMatrix(auto_create_col_mapper=True, auto_create_row_mapper=True) fileHandle = open(filePath, "r") numCells = 0 if header: fileHandle.readline() for line in fileHandle: numCells += 1 if (numCells % 1000000 == 0): print("Processed {} cells".format(numCells)) if (len(line)) > 1: line = line.split(separator) line[-1] = line[-1].replace("\n", "") try: user_id = line[0] item_id = line[1] try: value = float(line[2]) if value != 0.0: matrixBuilder.add_data_lists([user_id], [item_id], [value]) except ValueError: print( "load_CSV_into_SparseBuilder: Cannot parse as float value '{}'" .format(line[2])) except IndexError: print( "load_CSV_into_SparseBuilder: Index out of bound in line '{}'" .format(line)) fileHandle.close() return matrixBuilder.get_SparseMatrix( ), matrixBuilder.get_column_token_to_id_mapper( ), matrixBuilder.get_row_token_to_id_mapper()
def _loadURM(self): from Data_manager.IncrementalSparseMatrix import IncrementalSparseMatrix numCells = 0 URM_builder = IncrementalSparseMatrix(auto_create_col_mapper=True, auto_create_row_mapper=True) for current_split in [1, 2, 3, 4]: current_split_path = self.dataFile.extract( "combined_data_{}.txt".format(current_split), path=self.decompressed_zip_file_folder + "decompressed/") fileHandle = open(current_split_path, "r") print("NetflixPrizeReader: loading split {}".format(current_split)) currentMovie_id = None for line in fileHandle: if numCells % 1000000 == 0 and numCells != 0: print("Processed {} cells".format(numCells)) if (len(line)) > 1: line_split = line.split(",") # If line has 3 components, it is a 'user_id,rating,date' row if len(line_split) == 3 and currentMovie_id != None: user_id = line_split[0] URM_builder.add_data_lists([user_id], [currentMovie_id], [float(line_split[1])]) numCells += 1 # If line has 1 component, it MIGHT be a 'item_id:' row elif len(line_split) == 1: line_split = line.split(":") # Confirm it is a 'item_id:' row if len(line_split) == 2: currentMovie_id = line_split[0] else: print("Unexpected row: '{}'".format(line)) else: print("Unexpected row: '{}'".format(line)) fileHandle.close() print("NetflixPrizeReader: cleaning temporary files") shutil.rmtree(self.decompressed_zip_file_folder + "decompressed/", ignore_errors=True) return URM_builder.get_SparseMatrix( ), URM_builder.get_column_token_to_id_mapper( ), URM_builder.get_row_token_to_id_mapper()
def load_CSV_into_SparseBuilder (filePath, header = False, separator="::", timestamp = False, remove_duplicates = False, custom_user_item_rating_columns = None): URM_all_builder = IncrementalSparseMatrix(auto_create_col_mapper = True, auto_create_row_mapper = True) URM_timestamp_builder = IncrementalSparseMatrix(auto_create_col_mapper = True, auto_create_row_mapper = True) if timestamp: dtype={0:str, 1:str, 2:float, 3:float} columns = ['userId', 'itemId', 'interaction', 'timestamp'] else: dtype={0:str, 1:str, 2:float} columns = ['userId', 'itemId', 'interaction'] df_original = pd.read_csv(filepath_or_buffer=filePath, sep=separator, header= 0 if header else None, dtype=dtype, usecols=custom_user_item_rating_columns) # If the original file has more columns, keep them but ignore them df_original.columns = columns user_id_list = df_original['userId'].values item_id_list = df_original['itemId'].values interaction_list = df_original['interaction'].values # Check if duplicates exist num_unique_user_item_ids = df_original.drop_duplicates(['userId', 'itemId'], keep='first', inplace=False).shape[0] contains_duplicates_flag = num_unique_user_item_ids != len(user_id_list) if contains_duplicates_flag: if remove_duplicates: # # Remove duplicates. # This way of removing the duplicates keeping the last tiemstamp without removing other columns # would be the simplest, but it is so slow to the point of being unusable on any dataset but ML100k # idxs = df_original.groupby(by=['userId', 'itemId'], as_index=False)["timestamp"].idxmax() # df_original = df_original.loc[idxs] # Alternative faster way: # 1 - Sort in ascending order so that the last (bigger) timestamp is in the last position. Set Nan to be in the first position, to remove them if possible # 2 - Then remove duplicates for user-item keeping the last row, which will be the last timestamp. if timestamp: sort_by = ["userId", "itemId", "timestamp"] else: sort_by = ["userId", "itemId", 'interaction'] df_original.sort_values(by=sort_by, ascending=True, inplace=True, kind="quicksort", na_position="first") df_original.drop_duplicates(["userId", "itemId"], keep='last', inplace=True) user_id_list = df_original['userId'].values item_id_list = df_original['itemId'].values interaction_list = df_original['interaction'].values assert num_unique_user_item_ids == len(user_id_list), "load_CSV_into_SparseBuilder: duplicate (user, item) values found" else: assert num_unique_user_item_ids == len(user_id_list), "load_CSV_into_SparseBuilder: duplicate (user, item) values found" URM_all_builder.add_data_lists(user_id_list, item_id_list, interaction_list) if timestamp: timestamp_list = df_original['timestamp'].values URM_timestamp_builder.add_data_lists(user_id_list, item_id_list, timestamp_list) return URM_all_builder.get_SparseMatrix(), URM_timestamp_builder.get_SparseMatrix(), \ URM_all_builder.get_column_token_to_id_mapper(), URM_all_builder.get_row_token_to_id_mapper() return URM_all_builder.get_SparseMatrix(), \ URM_all_builder.get_column_token_to_id_mapper(), URM_all_builder.get_row_token_to_id_mapper()