def dataPreprocessiong(): logging.warning(u"运行日志:将从数据库中读出的数据进行数据处理") result_dr, result_course, result_user = getDataFromDB() drList = formatDataByType(SetType.SetType_List, result_dr) userList = formatDataByType(SetType.SetType_Set, result_user) courseList = formatDataByType(SetType.SetType_List, result_course) dr_length = len(drList) course_length = len(courseList) user_length = len(userList) user_mdic, user_mdicr = makeDic(userList) course_mdic, course_mdicr = makeDic(courseList) result, learned = [], [] for dr in drList: temp_result, temp_learned = [], [] #print dr[0] temp_result.append(user_mdic[dr[0]] + 1) temp_result.append(dr[1]) temp_result.append(dr[2] * 5) temp_learned.append(dr[0]) temp_learned.append(dr[1]) temp_learned.append(get_keys(dr[1], courseList)) result.append(temp_result) learned.append(temp_learned) data = pd.DataFrame(result) if drawRoc: saveBgInputMartix(data, user_mdicr) return data, learned, course_mdic, course_mdicr, user_mdic, user_mdicr, \ dr_length, course_length, user_length, courseList
def dataPreprocessiong(): result_dr, result_course, result_user = getDataFromDB() drList = formatDataByType(SetType.SetType_List, result_dr) userList = formatDataByType(SetType.SetType_Set, result_user) courseList = formatDataByType(SetType.SetType_List, result_course) dr_length = len(drList) course_length = len(courseList) user_length = len(userList) user_mdic, user_mdicr = makeDic(userList) course_mdic, course_mdicr = makeDic(courseList) result, learned = [], [] for dr in drList: temp_result, temp_learned = [], [] temp_result.append(user_mdic[dr[0]] + 1) temp_result.append(dr[1]) temp_result.append(dr[2] * 5) temp_learned.append(dr[0]) temp_learned.append(dr[1]) temp_learned.append(get_keys(dr[1], courseList)) result.append(temp_result) learned.append(temp_learned) data = pd.DataFrame(result) return data, learned, course_mdic, course_mdicr, user_mdic, user_mdicr, \ dr_length, course_length, user_length, courseList
def get_course_info_with_image(): dbHandle = DatabaseIo() if not dbHandle: return None # sql_course = "select id, name, image, description from course5000" sql_course = DataBaseQuery["interface_image"] # sql_classify = "select id, course_name, classify_name from course_classify5000" sql_classify = DataBaseQuery["classify_info"] result_course = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_course) result_course_classify = dbHandle.doSql( execType=DataBaseOperateType.SearchMany, sql=sql_classify) dbHandle.changeCloseFlag() courseList = formatDataByType(SetType.SetType_List, result_course) classifyList = formatDataByType(SetType.SetType_List, result_course_classify) return courseList, classifyList
def coll_main(): start = time.clock() # 获取数据 result_dr, result_course, result_user = cR.getDataFromDB() # 把从course_dr中读取出来的数据以列表形式存储 k = list() k = formatDataByType(SetType.SetType_List, result_dr) # 按course_id升序排序 result_list = sorted(k, key=lambda z: z[1]) # 读取user的id # 把从user_basic_info中读取出来的数据以列表形式存储 user_basic_info_list = formatDataByType(SetType.SetType_Set, result_user) # 把从course_info中读取出来的数据以列表形式存储 course_info_list = formatDataByType(SetType.SetType_List, result_course) course_length = len(result_course) user_length = len(result_user) range_length = len(result_dr) movies = result_course # 建立字典,实现课程id和索引序号之间的映射,方便后续工作 course_mdic, course_mdicr = makeDic(course_info_list) # 建立字典,实现用户id和索引序号之间的映射,方便后续工作 user_mdic, user_mdicr = makeDic(user_basic_info_list) ratings = list() for j in range(range_length): w = [] w.append(user_mdic[k[j][0]]) w.append(course_mdic[k[j][1]]) w.append(5 * k[j][2]) ratings.append(w) #print('ratings') #print(len(ratings)) demo = CF(movies, ratings, course_length, user_length, k=10) demo.recommendByUser() recommend_result = demo.recommandList print("训练集的数据为%d条" % (len(demo.trans_data))) print("测试集的数据为%d条" % (len(demo.test_data))) end = time.clock() print("耗费时间: %f s" % (end - start))
def loadData(): if FSLflag == False: all_data = pd.read_csv( '../DGL/ml-100k/u.data', sep='\t', header=None, names=['user_id', 'item_id', 'rating', 'timestamp']) # test_data = pd.read_csv('../DGL/ml-100k/ua.test', sep='\t', header=None, # names=['user_id', 'item_id', 'rating', 'timestamp']) user_data = pd.read_csv('../DGL/ml-100k/u.user', sep='|', header=None, encoding='latin1') item_data = pd.read_csv('../DGL/ml-100k/u.item', sep='|', header=None, encoding='latin1') # test_data = test_data[test_data['user_id'].isin(train_data['user_id']) & # test_data['item_id'].isin(train_data['item_id'])] # u_data = user_data[[0,1,2,3,4]] # u_data.columns = ['user_id','age','gender','occupation','zip_code'] # i_data = item_data # i_data.columns = ['item_id','title','release_date','video_release_date','IMDb_URL','unknown','Action','Adventure','Animation','Children', # 'Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller', # 'War','Western'] return all_data, user_data, item_data else: dbHandle = DatabaseIo() if not dbHandle: return None sql_dr = DataBaseQuery["course_dr"] sql_course = DataBaseQuery["course_info"] sql_user = DataBaseQuery["user_id"] sql_classify = DataBaseQuery["classify_info"] result_dr = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_dr) result_course = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_course) result_classify = dbHandle.doSql( execType=DataBaseOperateType.SearchMany, sql=sql_classify) dbHandle.changeCloseFlag() result_user = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_user) drList = formatDataByType(SetType.SetType_List, result_dr) all_data = pd.DataFrame(list(drList)) all_data.columns = ['user_id', 'item_id', 'rating'] user_data = pd.DataFrame(list(result_user)) item_data = pd.DataFrame(list(result_course)) classify_data = pd.DataFrame(list(result_classify)) classify_data.columns = [ 'id', 'course_name', 'classify_name', 'classify_id' ] return all_data, user_data, item_data, classify_data
def get_couse_info(): # logging.warning(u"运行日志:获取课程信息") dbHandle = DatabaseIo() if not dbHandle: return None # sql_course = "select id, name from course5000" sql_course = DataBaseQuery["course_info"] result_course = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_course) dbHandle.changeCloseFlag() courseList = formatDataByType(SetType.SetType_List, result_course) return courseList
def get_couse_info_with_video(): # logging.warning(u"运行日志:获取课程信息") dbHandle = DatabaseIo() if not dbHandle: return None # 先用图片替代模拟视频 # sql_course = "select id, name, image, description from course5000" sql_course = DataBaseQuery["interface_video"] result_course = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_course) dbHandle.changeCloseFlag() courseList = formatDataByType(SetType.SetType_List, result_course) return courseList
def getdata(FSLflag): print("get data") if FSLflag == False: train_data = pd.read_csv( '../DGL/ml-100k/ua.base', sep='\t', header=None, names=['user_id', 'item_id', 'rating', 'timestamp']) test_data = pd.read_csv( '../DGL/ml-100k/ua.test', sep='\t', header=None, names=['user_id', 'item_id', 'rating', 'timestamp']) user_data = pd.read_csv('../DGL/ml-100k/u.user', sep='|', header=None, encoding='latin1') item_data = pd.read_csv('../DGL/ml-100k/u.item', sep='|', header=None, encoding='latin1') test_data = test_data[ test_data['user_id'].isin(train_data['user_id']) & test_data['item_id'].isin(train_data['item_id'])] train_data = train_data.values.tolist() test_data = test_data.values.tolist() user_data = user_data.values.tolist() item_data = item_data.values.tolist() # print item_data # print item_data return train_data, test_data, user_data, item_data else: dbHandle = DatabaseIo() if not dbHandle: return None sql_dr = DataBaseQuery["course_dr"] sql_course = DataBaseQuery["course_info"] sql_user = DataBaseQuery["user_id"] sql_classify = DataBaseQuery["classify_info"] result_dr = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_dr) result_course = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_course) result_classify = dbHandle.doSql( execType=DataBaseOperateType.SearchMany, sql=sql_classify) dbHandle.changeCloseFlag() result_user = dbHandle.doSql(execType=DataBaseOperateType.SearchMany, sql=sql_user) drList = formatDataByType(SetType.SetType_List, result_dr) user_data = formatDataByType(SetType.SetType_Set, result_user) item_data = formatDataByType(SetType.SetType_List, result_course) classify_data = formatDataByType(SetType.SetType_List, result_classify) dr_length = len(drList) testIDs = random.sample(range(1, dr_length), int(dr_length / 10)) data = pd.DataFrame(drList) train_data = [] test_data = [] for index, row in data.iterrows(): if (index + 1) in testIDs: test_data.append(row) else: train_data.append(row) return train_data, test_data, user_data, item_data, classify_data
def transToMatrix(p): return formatDataByType(SetType.SetType_Set, p)
def dataPreprocessiong(): result_dr, result_course, result_user = getDataFromDB() drList = formatDataByType(SetType.SetType_List, result_dr) userList = formatDataByType(SetType.SetType_Set, result_user) courseList = formatDataByType(SetType.SetType_List, result_course) dr_length = len(drList) course_length = len(courseList) user_length = len(userList) user_mdic, user_mdicr = makeDic(userList) course_mdic, course_mdicr = makeDic(courseList) result, learned = [], [] for dr in drList: temp_result, temp_learned = [], [] temp_result.append(user_mdic[dr[0]] + 1) temp_result.append(dr[1]) temp_result.append(dr[2] * 5) temp_learned.append(dr[0]) temp_learned.append(dr[1]) temp_learned.append(get_keys(dr[1], courseList)) result.append(temp_result) learned.append(temp_learned) data = pd.DataFrame(result) #用dcL删除某些结果里没有的课程 dc = pd.read_csv("roc/toGcn2.csv") dc.drop_duplicates(subset=['cid'], keep='first', inplace=True) dcL = list(dc['cid']) #保存二部图输入 myfile = codecs.open("gcn/bg_input.csv", mode="w", encoding='utf-8') data.drop_duplicates(subset=[0, 1], keep='first', inplace=True) data.reset_index(inplace=True) #删除结果里没有的课程 for i in range(len(data)): if data[1][i] not in dcL: data.drop(i, axis=0, inplace=True) #打印去重的看有多少课程。0用户,1课程 # aa=data.drop_duplicates(subset=[1],keep='first') # print(aa.reset_index()) data.drop('index', axis=1, inplace=True) datalist = data.values.tolist() for row in datalist: row[0] = user_mdicr[int(row[0]) - 1] myfile.write(str(row[0])) myfile.write(",") myfile.write(str(row[1])) myfile.write(",") myfile.write(str(int((row[2])))) myfile.write("\n") myfile.close() #构造二部图的输入数据的真实值矩阵realGraph,以及长id到短id(0开始)的字典 userListNew = list(data[0]) for i in range(len(userListNew)): userListNew[i] = user_mdicr[userListNew[i] - 1] userListNewTemp = sorted(list(set(userListNew))) courseListNew = list(data[1]) courseListNewTemp = sorted(list(set(courseListNew))) course_mdic_new = {} user_mdic_new = {} for i in range(len(userListNewTemp)): user_mdic_new[userListNewTemp[i]] = i for i in range(len(courseListNewTemp)): course_mdic_new[courseListNewTemp[i]] = i realGraph = nm.zeros((len(courseListNewTemp), len(userListNewTemp))) print(len(courseListNewTemp)) for i in range(len(data)): realGraph[[course_mdic_new[courseListNew[i]]], user_mdic_new[userListNew[i]]] = 1 nm.savetxt('roc/realGraph.txt', realGraph) nm.save('user_mdic_new.npy', user_mdic_new) nm.save('course_mdic_new.npy', course_mdic_new) return data, learned, course_mdic, course_mdicr, user_mdic, user_mdicr,\ dr_length, course_length, user_length, courseList