def preProcessByIncrement(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pr_number'))) print("before drop:", df.shape) df = df.copy(deep=True) df.drop(columns=['review_user_login'], inplace=True) df.drop_duplicates(['pr_number'], inplace=True) print("after drop:", df.shape) test_data = df """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ test_data_y = {} for pull_number in test_data.drop_duplicates(['pr_number'])['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers return test_data, test_data_y, convertDict
def preProcessBySlide(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """时间转为时间戳""" df['test'] = df['pr_created_at'] df['pr_created_at'] = df['pr_created_at'].apply( lambda x: time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S"))) """先对tag做拆分""" tagDict = dict(list(df.groupby('pull_number'))) print("before drop:", df.shape) df = df.copy(deep=True) df.drop(columns=['review_user_login', 'repo_full_name'], inplace=True) df.drop_duplicates(['pull_number', 'commit_sha', 'file_filename'], inplace=True) print("after drop:", df.shape) """对已经有的特征向量和标签做训练集的拆分""" train_data = df.loc[df['label'] == False].copy(deep=True) test_data = df.loc[df['label']].copy(deep=True) train_data.drop(columns=['label'], inplace=True) test_data.drop(columns=['label'], inplace=True) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} for pull_number in train_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers test_data_y = {} for pull_number in test_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" """空comment的review包含na信息,但作为结果集是有用的,所以只对训练集去掉na""" # """处理NAN""" # df.dropna(how='any', inplace=True) # df.reset_index(drop=True, inplace=True) # df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['pr_author', 'reviewer']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pull_number'))) """对已经有的特征向量和标签做训练集的拆分""" train_data = df.loc[df['label'] == False].copy(deep=True) test_data = df.loc[df['label']].copy(deep=True) train_data.drop(columns=['label'], inplace=True) test_data.drop(columns=['label'], inplace=True) """8ii处理NAN""" train_data.dropna(how='any', inplace=True) train_data.reset_index(drop=True, inplace=True) train_data.fillna(value='', inplace=True) """过滤掉评论时间在数据集时间范围内之后的数据""" # 结束时间:数据集pr最晚的创建时间 pr_created_time_data = train_data['pr_created_at'] end_time = max(pr_created_time_data.to_list()) train_data = train_data[train_data['comment_at'] <= end_time] train_data.reset_index(drop=True, inplace=True) test_data_y = {} for pull_number in test_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) test_data_y[pull_number] = reviewers train_data_y = {} for pull_number in train_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) train_data_y[pull_number] = reviewers return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcessByIncrement(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,时间跨度相当于都是测试集, 没有作用 """ """注意: 输入文件中已经带有列名了""" """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pull_number'))) """时间转为时间戳""" df['pr_created_at'] = df['pr_created_at'].apply( lambda x: time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S"))) print("before drop:", df.shape) df = df.copy(deep=True) df.drop(columns=['review_user_login', 'repo_full_name'], inplace=True) df.drop_duplicates(['pull_number', 'commit_sha', 'file_filename'], inplace=True) print("after drop:", df.shape) test_data = df """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ test_data_y = {} for pull_number in df.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers return test_data, test_data_y, convertDict
def preProcessBySlide(df, dates): """参数说明 df:读取的dataframe对象 dates:作为测试的年月四元组 """ """注意: 输入文件中已经带有列名了""" """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """创建时间转化为时间戳""" df['pr_created_at'] = df['pr_created_at'].apply(lambda x: time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S"))) df['pr_created_at'] = df['pr_created_at'] / (24 * 3600) """先对输入数据做精简 只留下感兴趣的数据""" df = df[['pr_number', 'pr_title', 'review_user_login', 'label', 'pr_created_at']].copy(deep=True) print("before filter:", df.shape) df.drop_duplicates(inplace=True) print("after filter:", df.shape) """对人名字做数字处理""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pr_number'))) """先尝试所有信息团在一起""" df = df[['pr_number', 'pr_title', 'label', 'pr_created_at']].copy(deep=True) df.drop_duplicates(inplace=True) df.reset_index(drop=True, inplace=True) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 textList = [] for row in df.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """初步尝试提取词干效果反而下降了 。。。。""" """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) textList.append(tempList) print(textList.__len__()) """对分词列表建立字典 并提取特征数""" dictionary = corpora.Dictionary(textList) print('词典:', dictionary) feature_cnt = len(dictionary.token2id) print("词典特征数:", feature_cnt) """根据词典建立语料库""" corpus = [dictionary.doc2bow(text) for text in textList] # print('语料库:', corpus) """语料库训练TF-IDF模型""" tfidf = models.TfidfModel(corpus) """再次遍历数据,形成向量,向量是稀疏矩阵的形式""" wordVectors = [] for i in range(0, df.shape[0]): wordVectors.append(dict(tfidf[dictionary.doc2bow(textList[i])])) """对已经有的本文特征向量和标签做训练集和测试集的拆分""" trainData_index = df.loc[df['label'] == False].index testData_index = df.loc[df['label'] == True].index """训练集""" train_data = [wordVectors[x] for x in trainData_index] """测试集""" test_data = [wordVectors[x] for x in testData_index] """填充为向量""" train_data = DataProcessUtils.convertFeatureDictToDataFrame(train_data, featureNum=feature_cnt) test_data = DataProcessUtils.convertFeatureDictToDataFrame(test_data, featureNum=feature_cnt) train_data['pr_number'] = list(df.loc[df['label'] == False]['pr_number']) test_data['pr_number'] = list(df.loc[df['label'] == True]['pr_number']) train_data['pr_created_at'] = list(df.loc[df['label'] == False]['pr_created_at']) test_data['pr_created_at'] = list(df.loc[df['label'] == True]['pr_created_at']) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} for pull_number in df.loc[df['label'] == False]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers test_data_y = {} for pull_number in df.loc[df['label'] == True]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers """train_data ,test_data 最后一列是pr number test_data_y 的形式是dict""" return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" t1 = datetime.now() """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pr_number'))) """先尝试所有信息团在一起""" df = df[['pr_number', 'pr_title', 'pr_body', 'label']].copy(deep=True) df.drop_duplicates(inplace=True) df.reset_index(drop=True, inplace=True) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 textList = [] """由于特殊性 PB算法的训练集不是dataFrame { p1:set1, p2:set2, ... } """ train_data = {} test_data = {} for row in df.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的number""" pr_num = getattr(row, 'pr_number') label = getattr(row, 'label') """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """初步尝试提取词干效果反而下降了 。。。。""" """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) """pull request的body""" pr_body = getattr(row, 'pr_body') pr_body_word_list = [x for x in FleshReadableUtils.word_list(pr_body) if x not in stopwords] """对单词做提取词干""" pr_body_word_list = nltkFunction.stemList(pr_body_word_list) tempList.extend(pr_body_word_list) wordSet = MultisetHelper.WordMultiset() wordSet.add(tempList) if label == 0: train_data[pr_num] = wordSet else: test_data[pr_num] = wordSet print("train size:", train_data.items().__len__()) print("test size:", test_data.items().__len__()) """问题转化为多标签问题 train_data_y [{pull_number:[(r1, s1), (r2, s2), ...]}, ... ,{}] r 代表reviewer s 代表集合 """ train_data_y = {} for pull_number in df.loc[df['label'] == False]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) tempDf = tagDict[pull_number][['review_user_login', 'comment_body']].copy(deep=True) commentDict = dict(list(tempDf.groupby('review_user_login'))) reviewerList = [] for reviewer in reviewers: commentDf = commentDict[reviewer] wordSet = MultisetHelper.WordMultiset() for row in commentDf.itertuples(index=False, name='Pandas'): comment = getattr(row, 'comment_body') comment_body_word_list = [x for x in FleshReadableUtils.word_list(comment) if x not in stopwords] """对单词做提取词干""" comment_body_word_list = nltkFunction.stemList(comment_body_word_list) wordSet.add(comment_body_word_list) reviewerList.append((reviewer, wordSet)) train_data_y[pull_number] = reviewerList test_data_y = {} for pull_number in df.loc[df['label'] == True]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) tempDf = tagDict[pull_number][['review_user_login', 'comment_body']].copy(deep=True) commentDict = dict(list(tempDf.groupby('review_user_login'))) reviewerList = [] for reviewer in reviewers: commentDf = commentDict[reviewer] wordSet = MultisetHelper.WordMultiset() for row in commentDf.itertuples(index=False, name='Pandas'): comment = getattr(row, 'comment_body') comment_body_word_list = [x for x in FleshReadableUtils.word_list(comment) if x not in stopwords] """对单词做提取词干""" comment_body_word_list = nltkFunction.stemList(comment_body_word_list) wordSet.add(comment_body_word_list) reviewerList.append((reviewer, wordSet)) test_data_y[pull_number] = reviewerList print("preprocess cost time:", datetime.now() - t1) return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" """空comment的review包含na信息,但作为结果集是有用的,所以只对训练集去掉na""" # """处理NAN""" # df.dropna(how='any', inplace=True) # df.reset_index(drop=True, inplace=True) df['pr_title'].fillna(value='', inplace=True) df['pr_body'].fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['pr_author', 'reviewer']) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 """问题:lsi的过程不能在整个数据集上面做,不然会导致pr的文本重复问题""" df_pr = df.copy(deep=True) df_pr.drop_duplicates(subset=['pull_number'], keep='first', inplace=True) df_pr.reset_index(drop=True, inplace=True) # 用于记录pr中文字的数量,对于pr少于10个word的pr.直接去掉 df_pr_word_count = [] textList = [] for row in df_pr.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """初步尝试提取词干效果反而下降了 。。。。""" """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) """pull request的body""" pr_body = getattr(row, 'pr_body') pr_body_word_list = [x for x in FleshReadableUtils.word_list(pr_body) if x not in stopwords] """对单词做提取词干""" pr_body_word_list = nltkFunction.stemList(pr_body_word_list) tempList.extend(pr_body_word_list) if tempList.__len__() >= 10 or getattr(row, 'label'): textList.append(tempList) if getattr(row, 'label'): df_pr_word_count.append(10) # 以便过后面的过滤 else: df_pr_word_count.append(tempList.__len__()) """去除无用的训练pr""" df_pr['count'] = df_pr_word_count df_pr = df_pr.loc[df_pr['count'] >= 10].copy(deep=True) df_pr.reset_index(drop=True, inplace=True) df_pr.drop(['count'], inplace=True, axis=1) """保存只有pr的列表""" prList = list(df_pr['pull_number']) """对已经有的本文特征向量和标签做训练集和测试集的拆分""" trainData_index = df_pr.loc[df_pr['label'] == False].index testData_index = df_pr.loc[df_pr['label'] == True].index trainDataTextList = [textList[x] for x in trainData_index] testDataTextList = [textList[x] for x in testData_index] print(textList.__len__()) """对分词列表建立字典 并提取特征数""" dictionary = corpora.Dictionary(trainDataTextList) print('词典:', dictionary) """感觉有问题,tfidf模型不应该是在全数据集上面计算,而是在训练集上面计算,而测试集的向量就是 单纯的带入模型的计算结果""" """根据词典建立语料库""" corpus = [dictionary.doc2bow(text) for text in trainDataTextList] # print('语料库:', corpus) """语料库训练TF-IDF模型""" tfidf = models.TfidfModel(corpus) corpus_tfidf = tfidf[corpus] topic_num = 10 lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=topic_num) topic_list = lsi.print_topics() print("{0}个主题的单词分布为:\n".format(topic_num)) for topic in topic_list: print(topic) """再次遍历数据,形成向量,向量是稀疏矩阵的形式""" wordVectors = [] for i in range(0, trainDataTextList.__len__()): wordVectors.append(dict(lsi[dictionary.doc2bow(trainDataTextList[i])])) for i in range(0, testDataTextList.__len__()): wordVectors.append(dict(lsi[dictionary.doc2bow(testDataTextList[i])])) """训练集""" train_data = [wordVectors[x] for x in trainData_index] """测试集""" test_data = [wordVectors[x] for x in testData_index] """填充为向量""" train_v_data = DataProcessUtils.convertFeatureDictToDataFrame(train_data, featureNum=topic_num) test_v_data = DataProcessUtils.convertFeatureDictToDataFrame(test_data, featureNum=topic_num) lsi_data = pandas.concat([train_v_data, test_v_data], axis=0) # 0 轴合并 lsi_data['pull_number'] = prList lsi_data.reset_index(inplace=True, drop=True) train_data = df.loc[df['label'] == False] train_data.reset_index(drop=True, inplace=True) test_data = df.loc[df['label'] == True] test_data.reset_index(drop=True, inplace=True) train_data = train_data.merge(lsi_data, on="pull_number") train_data.drop(columns=['label'], inplace=True) test_data = test_data.merge(lsi_data, on="pull_number") test_data.drop(columns=['label'], inplace=True) """8ii处理NAN""" train_data.dropna(how='any', inplace=True) train_data.reset_index(drop=True, inplace=True) train_data.fillna(value='', inplace=True) """先对tag做拆分""" trainDict = dict(list(train_data.groupby('pull_number'))) testDict = dict(list(test_data.groupby('pull_number'))) test_data_y = {} for pull_number in test_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(testDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) test_data_y[pull_number] = reviewers train_data_y = {} for pull_number in train_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(trainDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) train_data_y[pull_number] = reviewers return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, date, project, isSTD=False, isNOR=False): """参数说明 df:读取的dataframe对象 testDate:作为测试的年月 (year,month) isSTD:对数据是否标准化 isNOR:对数据是否归一化 """ print("start df shape:", df.shape) """过滤NA的数据""" df.dropna(axis=0, how='any', inplace=True) print("after fliter na:", df.shape) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == date[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == date[3])) df.reset_index(drop=True, inplace=True) """对人名字做数字处理""" """频率不过的评审者在编号之前就已经过滤了,不用考虑分类不连续的情况""" """这里reviewer_user_login 放在 第一个否则会影响candicateNum这个变量在后面的引用""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login', 'author_user_login']) recoverDict = {v: k for k, v in convertDict.items()} print(df.shape) candicateNum = max(df.loc[df['label'] == 0]['review_user_login']) print("candicate Num:", candicateNum) """先对输入数据做精简 只留下感兴趣的数据""" df = df[['pr_number', 'review_user_login', 'author_user_login', 'author_association', 'commits', 'deletions', 'additions', 'changed_files', 'label', 'merged']].copy(deep=True) print("before filter:", df.shape) df.drop_duplicates(inplace=True) print("after filter:", df.shape) """计算作者的关系""" df['author_association'] = df['author_association'].apply(lambda x: x == 'MEMBER') """计算累积的历史数据""" request_number_prs = [] # 作者之前发出的数量 request_number_merged_prs = [] # 作者发出的被接受的数量 request_number_rejected_prs = [] # 作者发出被拒绝的数量 request_accept_rate = [] # 作者pr被接受的概率 request_reject_rate = [] # 作者pr被拒绝的概率 for row in df.itertuples(): pr_num = getattr(row, 'pr_number') author = getattr(row, 'author_user_login') """过滤历史的pr""" temp_df = df.loc[(df['pr_number'] < pr_num)&(df['author_user_login'] == author)] request_number_prs.append(temp_df.shape[0]) accept_times = temp_df.loc[temp_df['merged'] == 1].shape[0] request_number_merged_prs.append(accept_times) request_number_rejected_prs.append(temp_df.shape[0] - accept_times) if temp_df.shape[0] > 0: request_accept_rate.append(accept_times/temp_df.shape[0]) request_reject_rate.append(1 - accept_times / temp_df.shape[0]) else: request_accept_rate.append(0) request_reject_rate.append(0) df['request_number_prs'] = request_number_prs df['request_number_merged_prs'] = request_number_merged_prs df['request_number_rejected_prs'] = request_number_rejected_prs df['request_accept_rate'] = request_accept_rate df['request_reject_rate'] = request_reject_rate """添加作者是否关注项目""" user_watch_repo_relation_path = projectConfig.getUserWatchRepoRelation() userWatchRepoRelation = pandasHelper.readTSVFile( os.path.join(user_watch_repo_relation_path, f'userWatchRepoRelation.tsv'), pandasHelper.INT_READ_FILE_WITH_HEAD, low_memory=False ) watchRepoMap = {} for k in convertDict.keys(): """获取 reviewer 的 follow 列表""" following_list = list(set(userWatchRepoRelation.loc[userWatchRepoRelation['login'] == k]['repo_full_name'])) isFollow = False for repo in following_list: owner, name = repo.split('/') if name == project: isFollow = True watchRepoMap[convertDict[k]] = isFollow request_watches = [] for row in df.itertuples(): author = getattr(row, 'author_user_login') request_watches.append(watchRepoMap[author]) df['request_watches'] = request_watches """添加作者follower数量, followings数量, 是否follow团队成员""" user_follow_relation_path = projectConfig.getUserFollowRelation() userFollowRelation = pandasHelper.readTSVFile( os.path.join(user_follow_relation_path, f'userFollowRelation.tsv'), pandasHelper.INT_READ_FILE_WITH_HEAD, low_memory=False ) followMap = {} followerCountMap = {} followingCountMap = {} followCoreMemberMap = {} """收集核心成员列表""" coreMemberList = list(set(df.loc[df['author_association'] == 1]['author_user_login'])) for k in convertDict.keys(): """获取 reviewer 的 follow 列表""" following_list = list(set(userFollowRelation.loc[userFollowRelation['login'] == k]['following_login'])) followingCountMap[convertDict[k]] = following_list.__len__() isFollowCoreMember = False for f in following_list: if f in convertDict.keys(): followMap[(convertDict[k], convertDict[f])] = 1 if f in coreMemberList: isFollowCoreMember = True followCoreMemberMap[convertDict[k]] = isFollowCoreMember follower_list = list(set(userFollowRelation.loc[userFollowRelation['following_login'] == k]['login'])) followerCountMap[convertDict[k]] = follower_list.__len__() # for f in follower_list: # if f in convertDict.keys(): # followMap[(convertDict[f], convertDict[k])] = 1 request_number_follows = [] request_number_following = [] request_follow_ct = [] for row in df.itertuples(): pr_num = getattr(row, 'pr_number') author = getattr(row, 'author_user_login') """过滤历史的pr""" request_number_following.append(followingCountMap[author]) request_number_follows.append(followerCountMap[author]) request_follow_ct.append(followCoreMemberMap[author]) df['request_number_following'] = request_number_following df['request_number_follows'] = request_number_follows df['request_follow_ct'] = request_follow_ct """先提前统计正确答案""" tagDict = dict(list(df.groupby('pr_number'))) train_data = df.loc[df['label'] == 0].copy(deep=True) test_data = df.loc[df['label'] == 1].copy(deep=True) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} pull_number_list = train_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in pull_number_list: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers train_data.drop(columns=['review_user_login', 'author_user_login', 'label', 'merged'], inplace=True) train_data.drop_duplicates(inplace=True) train_data.drop_duplicates(subset=['pr_number'], inplace=True) train_data.drop(columns=['pr_number'], inplace=True) """训练集 结果做出多标签分类通用的模式""" train_data_y = DataProcessUtils.convertLabelListToDataFrame(train_data_y, pull_number_list, candicateNum) test_data_y = {} pull_number_list = test_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in test_data.drop_duplicates(['pr_number'])['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers test_data.drop(columns=['review_user_login', 'author_user_login', 'label', 'merged'], inplace=True) test_data.drop_duplicates(subset=['pr_number'], inplace=True) """获得pr list""" prList = list(test_data['pr_number']) test_data.drop(columns=['pr_number'], inplace=True) test_data_y = DataProcessUtils.convertLabelListToListArray(test_data_y, pull_number_list) """参数规范化""" if isSTD: stdsc = StandardScaler() train_data_std = stdsc.fit_transform(train_data) test_data_std = stdsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList elif isNOR: maxminsc = MinMaxScaler() train_data_std = maxminsc.fit_transform(train_data) test_data_std = maxminsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList else: return train_data, train_data_y, test_data, test_data_y, convertDict, prList
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:作为测试的年月四元组 """ """注意: 输入文件中已经带有列名了""" """issue comment 和 review comment关注的""" """处理NAN""" # df.dropna(how='any', inplace=True) # df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对人名字做数字处理""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login', 'author_user_login']) df['pr_created_at'] = df['pr_created_at'].apply(lambda x: time.strptime(x, "%Y-%m-%d %H:%M:%S")) """对 comment_at 处理增加具体天数的标识""" df['day'] = df['pr_created_at'].apply(lambda x: 10000 * x.tm_year + 100 * x.tm_mon + x.tm_mday) # 20200821 """先对tag做拆分""" temp_df = df.copy(deep=True) temp_df.drop(columns=['filename'], inplace=True) temp_df.drop_duplicates(inplace=True) tagDict = dict(list(temp_df.groupby('pr_number'))) """先尝试所有信息团在一起""" df = df[['pr_number', 'filename', 'label']].copy(deep=True) df.drop_duplicates(inplace=True) df.reset_index(drop=True, inplace=True) """对已经有的特征向量和标签做训练集的拆分""" train_data = df.loc[df['label'] == False].copy(deep=True) test_data = df.loc[df['label']].copy(deep=True) train_data.drop(columns=['label'], inplace=True) test_data.drop(columns=['label'], inplace=True) """问题转化为多标签问题 train_data_y [{pull_number:[(r1, d1), (r2, d2), ...]}, ... ,{}] """ """训练集存的是作者 测试集存的是评审者""" train_data_y = {} for pull_number in df.loc[df['label'] == False]['pr_number']: tempDf = tagDict[pull_number] author = [] for row in tempDf.itertuples(index=False, name='Pandas'): a = getattr(row, 'author_user_login') day = getattr(row, 'day') author.append((a, None, day)) break train_data_y[pull_number] = author test_data_y = {} for pull_number in df.loc[df['label'] == True]['pr_number']: tempDf = tagDict[pull_number] reviewers = [] for row in tempDf.itertuples(index=False, name='Pandas'): r = getattr(row, 'review_user_login') comment_node_id = getattr(row, 'comment_node_id') day = getattr(row, 'day') reviewers.append((r, comment_node_id, day)) test_data_y[pull_number] = reviewers """train_data ,test_data 最后一列是pr number test_data_y 的形式是dict""" return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, date, project, featureType, isSTD=False, isNOR=False): """参数说明 df:读取的dataframe对象 testDate:作为测试的年月 (year,month) isSTD:对数据是否标准化 isNOR:对数据是否归一化 """ print("start df shape:", df.shape) """过滤NA的数据""" df.dropna(axis=0, how='any', inplace=True) print("after fliter na:", df.shape) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == date[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == date[3])) df.sort_values(by='pr_number', ascending=True, inplace=True) df.reset_index(drop=True, inplace=True) # """在现有的特征中添加文本路径特征""" """更正说明:由于PCA不能训练集和测试集同时降维,否则相当于使用了后面的信息 所以添加之前必须两者分别处理 4.13 append 函数必须在表明label后面使用""" if featureType == 1 or featureType == 3: df = appendFilePathFeatureVector(df, project, date, 'pr_number') """在现有的特征中添加pr标题和内容文本特征""" if featureType == 2 or featureType == 3: df = appendTextualFeatureVector(df, project, date, 'pr_number') # """频率统计每一个reviewer的次数,排除数量过少的reviewer""" # freq = {} # for data in df.itertuples(index=False): # name = data[list(df.columns).index('review_user_login')] # if freq.get(name, None) is None: # freq[name] = 0 # """训练集用户次数加一 测试集直接保留 """ # if not data[list(df.columns).index('label')]: # freq[name] += 1 # else: # freq[name] += 1 # # num = 5 # df['freq'] = df['review_user_login'].apply(lambda x: freq[x]) # df = df.loc[df['freq'] > num].copy(deep=True) # df.drop(columns=['freq'], inplace=True) # df.reset_index(drop=True, inplace=True) # print("after lifter unexperienced user:"******"""对人名字做数字处理""" """频率不过的评审者在编号之前就已经过滤了,不用考虑分类不连续的情况""" """这里reviewer_user_login 放在 第一个否则会影响candicateNum这个变量在后面的引用""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login', 'pr_user_login']) print(df.shape) candicateNum = max(df.loc[df['label'] == 0]['review_user_login']) print("candicate Num:", candicateNum) """对branch做处理 舍弃base,head做拆分 并数字化""" df.drop(axis=1, columns=['pr_base_label'], inplace=True) # inplace 代表直接数据上面 df['pr_head_tail'] = df['pr_head_label'] df['pr_head_tail'] = df['pr_head_tail'].apply(lambda x: x.split(':')[1]) df['pr_head_label'] = df['pr_head_label'].apply(lambda x: x.split(':')[0]) df.drop(axis=1, columns=['pr_head_tail'], inplace=True) # MLTrain.changeStringToNumber(df, ['pr_head_tail']) DataProcessUtils.changeStringToNumber(df, ['pr_head_label']) """时间转时间戳处理""" df['pr_created_at'] = df['pr_created_at'].apply( lambda x: int(time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S")))) """先对tag做拆分""" tagDict = dict(list(df.groupby('pr_number'))) """对已经有的特征向量和标签做训练集的拆分""" train_data = df.loc[df['label'] == False].copy(deep=True) test_data = df.loc[df['label']].copy(deep=True) train_data.drop(columns=['label'], inplace=True) test_data.drop(columns=['label'], inplace=True) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} pull_number_list = train_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in pull_number_list: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers train_data.drop(columns=['review_user_login'], inplace=True) train_data.drop_duplicates(inplace=True) """训练集 结果做出多标签分类通用的模式""" train_data_y = DataProcessUtils.convertLabelListToDataFrame(train_data_y, pull_number_list, candicateNum) test_data_y = {} pull_number_list = test_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in test_data.drop_duplicates(['pr_number'])['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers test_data.drop(columns=['review_user_login'], inplace=True) test_data.drop_duplicates(inplace=True) # test_data_y = DataProcessUtils.convertLabelListToDataFrame(test_data_y, pull_number_list, candicateNum) test_data_y = DataProcessUtils.convertLabelListToListArray(test_data_y, pull_number_list) """获得pr list""" prList = list(test_data['pr_number']) """去除pr number""" test_data.drop(columns=['pr_number'], inplace=True) train_data.drop(columns=['pr_number'], inplace=True) """参数规范化""" if isSTD: stdsc = StandardScaler() train_data_std = stdsc.fit_transform(train_data) test_data_std = stdsc.transform(test_data) # print(train_data_std) # print(test_data_std.shape) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList elif isNOR: maxminsc = MinMaxScaler() train_data_std = maxminsc.fit_transform(train_data) test_data_std = maxminsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList else: return train_data, train_data_y, test_data, test_data_y, convertDict, prList
def preProcess(df, date, project, isSTD=False, isNOR=False): """参数说明 df:读取的dataframe对象 testDate:作为测试的年月 (year,month) isSTD:对数据是否标准化 isNOR:对数据是否归一化 """ print("start df shape:", df.shape) """过滤NA的数据""" df.dropna(axis=0, how='any', inplace=True) print("after fliter na:", df.shape) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == date[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == date[3])) df.reset_index(drop=True, inplace=True) """对人名字做数字处理""" """频率不过的评审者在编号之前就已经过滤了,不用考虑分类不连续的情况""" """这里reviewer_user_login 放在 第一个否则会影响candicateNum这个变量在后面的引用""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) print(df.shape) candicateNum = max(df.loc[df['label'] == 0]['review_user_login']) print("candicate Num:", candicateNum) """先对输入数据做精简 只留下感兴趣的数据""" df = df[['pr_number', 'pr_title', 'pr_body', 'review_user_login', 'label']].copy(deep=True) print("before filter:", df.shape) df.drop_duplicates(inplace=True) print("after filter:", df.shape) """先提前统计正确答案""" tagDict = dict(list(df.groupby('pr_number'))) train_data = df.loc[df['label'] == 0].copy(deep=True) test_data = df.loc[df['label'] == 1].copy(deep=True) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} pull_number_list = train_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in pull_number_list: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers train_data.drop(columns=['review_user_login'], inplace=True) train_data.drop_duplicates(inplace=True) train_data.drop_duplicates(subset=['pr_number'], inplace=True) """训练集 结果做出多标签分类通用的模式""" train_data_y = DataProcessUtils.convertLabelListToDataFrame(train_data_y, pull_number_list, candicateNum) test_data_y = {} pull_number_list = test_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in test_data.drop_duplicates(['pr_number'])['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers test_data.drop(columns=['review_user_login'], inplace=True) test_data.drop_duplicates(inplace=True) """pr_number 经过去重""" test_data.drop_duplicates(subset=['pr_number'], inplace=True) # test_data_y = DataProcessUtils.convertLabelListToDataFrame(test_data_y, pull_number_list, candicateNum) test_data_y = DataProcessUtils.convertLabelListToListArray(test_data_y, pull_number_list) """获得pr list""" prList = list(test_data['pr_number']) """先尝试所有信息团在一起""" df = df[['pr_number', 'pr_title', 'pr_body', 'label']].copy(deep=True) df.drop_duplicates(inplace=True) df.reset_index(drop=True, inplace=True) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 textList = [] for row in df.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """初步尝试提取词干效果反而下降了 。。。。""" """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) """pull request的body""" pr_body = getattr(row, 'pr_body') pr_body_word_list = [x for x in FleshReadableUtils.word_list(pr_body) if x not in stopwords] """对单词做提取词干""" pr_body_word_list = nltkFunction.stemList(pr_body_word_list) tempList.extend(pr_body_word_list) textList.append(tempList) print(textList.__len__()) """对分词列表建立字典 并提取特征数""" dictionary = corpora.Dictionary(textList) print('词典:', dictionary) feature_cnt = len(dictionary.token2id) print("词典特征数:", feature_cnt) """根据词典建立语料库""" corpus = [dictionary.doc2bow(text) for text in textList] # print('语料库:', corpus) """语料库训练TF-IDF模型""" tfidf = models.TfidfModel(corpus) """再次遍历数据,形成向量,向量是稀疏矩阵的形式""" wordVectors = [] for i in range(0, df.shape[0]): wordVectors.append(dict(tfidf[dictionary.doc2bow(textList[i])])) """对已经有的本文特征向量和标签做训练集和测试集的拆分""" trainData_index = df.loc[df['label'] == False].index testData_index = df.loc[df['label'] == True].index """训练集""" train_data = [wordVectors[x] for x in trainData_index] """测试集""" test_data = [wordVectors[x] for x in testData_index] """填充为向量""" train_data = DataProcessUtils.convertFeatureDictToDataFrame(train_data, featureNum=feature_cnt) test_data = DataProcessUtils.convertFeatureDictToDataFrame(test_data, featureNum=feature_cnt) """参数规范化""" if isSTD: stdsc = StandardScaler() train_data_std = stdsc.fit_transform(train_data) test_data_std = stdsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList elif isNOR: maxminsc = MinMaxScaler() train_data_std = maxminsc.fit_transform(train_data) test_data_std = maxminsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList else: return train_data, train_data_y, test_data, test_data_y, convertDict, prList
def preProcess(df, date, project, isSTD=False, isNOR=False, m=3): """参数说明 df:读取的dataframe对象 testDate:作为测试的年月 (year,month) isSTD:对数据是否标准化 isNOR:对数据是否归一化 m: 超参数,窗口时间 """ print("start df shape:", df.shape) """过滤NA的数据""" df.dropna(axis=0, how='any', inplace=True) print("after fliter na:", df.shape) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == date[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == date[3])) df['label_y'] = df['pr_created_at'].apply(lambda x: time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year) df['label_m'] = df['pr_created_at'].apply(lambda x: time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon) df.reset_index(drop=True, inplace=True) """更正说明:由于PCA不能训练集和测试集同时降维,否则相当于使用了后面的信息 所以添加之前必须两者分别处理 4.13 append 函数必须在表明label后面使用""" """添加File Path Features""" df = appendFilePathFeatureVector(df, project, date, 'pr_number') """读取User Follow的信息""" user_follow_relation_path = projectConfig.getUserFollowRelation() userFollowRelation = pandasHelper.readTSVFile( os.path.join(user_follow_relation_path, f'userFollowRelation.tsv'), pandasHelper.INT_READ_FILE_WITH_HEAD, low_memory=False ) def isInTimeGap(x, m, maxYear, maxMonth): d = x['label_y'] * 12 + x['label_m'] d2 = maxYear * 12 + maxMonth return d >= d2 - m """对人名字做数字处理""" """频率不过的评审者在编号之前就已经过滤了,不用考虑分类不连续的情况""" """这里reviewer_user_login 放在 第一个否则会影响candicateNum这个变量在后面的引用""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login', 'pr_user_login']) print(df.shape) candicateNum = max(df.loc[df['label'] == 0]['review_user_login']) print("candicate Num:", candicateNum) """计算contributor set""" contribute_list = list(set(df.loc[df['label'] == 1]['pr_user_login'])) reviewer_list = list(set(df.loc[df['label'] == 0]['review_user_login'])) """添加Relation ship Features""" """对 train set和test set的处理方式稍微不同 train set数据统计依照之前pr 而训练集的统计数据只限制于trianset """ """把 df 的pr_created_at 和 comment_at 转化为时间戳""" df['pr_created_at'] = df['pr_created_at'].apply( lambda x: time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S"))) df['comment_at'] = df['comment_at'].apply(lambda x: time.mktime(time.strptime(x, "%Y-%m-%d %H:%M:%S"))) df['response_time'] = df['comment_at'] - df['pr_created_at'] """Prior Evaluation reviewer cm 之前 review co的次数 Recent Evaluation reviewer cm 在 m 个月 reivew co的次数 Follow Relation co 是否follow cm Follower Relation cm 是否follow co """ startTime = datetime.now() prior_evaluation = {} recent_evaluation = {} follower_relation = {} following_relation = {} followMap = {} for k in convertDict.keys(): """获取 reviewer 的 follow 列表""" follower_list = list(set(userFollowRelation.loc[userFollowRelation['login'] == k]['following_login'])) for f in follower_list: if f in convertDict.keys(): followMap[(convertDict[k], convertDict[f])] = 1 for reviewer in reviewer_list: prior_evaluation[reviewer] = [] recent_evaluation[reviewer] = [] follower_relation[reviewer] = [] following_relation[reviewer] = [] cols = list(df.columns) for data in df.itertuples(index=False, name='Pandas'): if data.__len__() < 14: pullNumber = getattr(data, 'pr_number') author = getattr(data, 'pr_user_login') label = getattr(data, 'label') label_m = getattr(data, 'label_m') label_y = getattr(data, 'label_y') else: pullNumber = data[cols.index("pr_number")] author = data[cols.index("pr_user_login")] label = data[cols.index("label")] label_m = data[cols.index("label_m")] label_y = data[cols.index("label_y")] temp = None if label == 0: temp = df.loc[df['pr_number'] < pullNumber] else: temp = df.loc[df['label'] == 0] temp = temp.loc[df['pr_user_login'] == author].copy(deep=True) """依次遍历每个候选者统计""" prior_evaluation_dict = dict(temp['review_user_login'].value_counts()) for r in reviewer_list: prior_evaluation[r].append(prior_evaluation_dict.get(r, 0)) """temp 二次过滤 选m个月以内的""" if temp.shape[0] > 0: if label == 0: temp['target'] = temp.apply(lambda x: isInTimeGap(x, m, label_y, label_m), axis=1) else: temp['target'] = temp.apply(lambda x: isInTimeGap(x, m, date[2], date[3]), axis=1) temp = temp.loc[temp['target'] == 1] """依次遍历每个候选者统计""" recent_evaluation_dict = dict(temp['review_user_login'].value_counts()) for r in reviewer_list: recent_evaluation[r].append(recent_evaluation_dict.get(r, 0)) """添加 follow 和 following 信息""" for r in reviewer_list: follower_relation[r].append(followMap.get((author, r), 0)) following_relation[r].append(followMap.get((r, author), 0)) """添加""" for r in reviewer_list: df[f'prior_evaluation_{r}'] = prior_evaluation[r] df[f'recent_evaluation_{r}'] = recent_evaluation[r] df[f'follower_relation_{r}'] = follower_relation[r] df[f'following_relation_{r}'] = following_relation[r] print("prior cost time:", datetime.now() - startTime) startTime = datetime.now() # 开始时间:数据集开始时间的前一天 start_time = time.strptime(str(date[0]) + "-" + str(date[1]) + "-" + "01 00:00:00", "%Y-%m-%d %H:%M:%S") start_time = int(time.mktime(start_time) - 86400) # 结束时间:数据集的最后一天 end_time = time.strptime(str(date[2]) + "-" + str(date[3]) + "-" + "01 00:00:00", "%Y-%m-%d %H:%M:%S") end_time = int(time.mktime(end_time) - 1) """Activeness Feature 添加""" total_pulls = {} # 项目有的所有pr evaluate_pulls = {} # co 之前review的数量 recent_pulls = {} # co 最近m月 review的数量 evaluate_time = {} # co 平均回应时间 last_time = {} # co 最后一次reivew 的时间间隔 first_time = {} # co 第一次review的时间间隔 for reviewer in reviewer_list: total_pulls[reviewer] = [] evaluate_pulls[reviewer] = [] recent_pulls[reviewer] = [] evaluate_time[reviewer] = [] last_time[reviewer] = [] first_time[reviewer] = [] count = 0 cols = list(df.columns) index_pr_number = cols.index("pr_number") index_pr_label = cols.index("label") index_pr_label_m = cols.index("label_m") index_pr_label_y = cols.index("label_y") for data in df.itertuples(index=False): print("count for active:", count) count += 1 pullNumber = data[index_pr_number] label = data[index_pr_label] label_m = data[index_pr_label_m] label_y = data[index_pr_label_y] temp = None if label == 0: temp = df.loc[df['pr_number'] < pullNumber].copy(deep=True) else: temp = df.loc[df['label'] == 0].copy(deep=True) """依次遍历每个候选者统计""" total_pull_number = list(set(temp['pr_number'])).__len__() res_reviewer_list = reviewer_list.copy() groups = dict(list(temp.groupby('review_user_login'))) """先遍历有tempDf的reviewer""" for r, tempDf in groups.items(): total_pulls[r].append(total_pull_number) res_reviewer_list.remove(r) if tempDf.shape[0] == 0: """没有历史 认为age=0, 间隔是最大间隔""" first_time[r].append(0) last_time[r].append(end_time - start_time) else: pr_created_time_list = list(tempDf['pr_created_at']) first_review_time = min(pr_created_time_list) last_review_time = max(pr_created_time_list) first_time[r].append(end_time - first_review_time) last_time[r].append(end_time - last_review_time) evaluate_pulls[r].append(tempDf.shape[0]) """平均回应时间统计""" if tempDf.shape[0] > 0: evaluate_avg = sum(tempDf['response_time']) evaluate_avg /= tempDf.shape[0] else: evaluate_avg = end_time - start_time evaluate_time[r].append(evaluate_avg) for r in res_reviewer_list: total_pulls[r].append(total_pull_number) evaluate_pulls[r].append(0) first_time[r].append(0) last_time[r].append(end_time - start_time) evaluate_avg = end_time - start_time evaluate_time[r].append(evaluate_avg) # recent_pulls[r].append(0) """过滤k个月 重新计算""" if label == 0: if temp.shape[0] > 0: temp['target'] = temp.apply(lambda x: isInTimeGap(x, m, label_y, label_m), axis=1) temp = temp.loc[temp['target'] == 1] else: if temp.shape[0] > 0: temp['target'] = temp.apply(lambda x: isInTimeGap(x, m, date[2], date[3]), axis=1) temp = temp.loc[temp['target'] == 1] res_reviewer_list = reviewer_list.copy() groups = dict(list(temp.groupby('review_user_login'))) """先遍历有tempDf的reviewer""" for r, tempDf in groups.items(): recent_pulls[r].append(tempDf.shape[0]) res_reviewer_list.remove(r) for r in res_reviewer_list: recent_pulls[r].append(0) """Activeness Feature增加到 dataframe""" for r in reviewer_list: df[f'total_pulls_{r}'] = total_pulls[r] df[f'evaluate_pulls_{r}'] = evaluate_pulls[r] df[f'recent_pulls_{r}'] = recent_pulls[r] df[f'first_time_{r}'] = first_time[r] df[f'last_time_{r}'] = last_time[r] df[f'evaluate_time_{r}'] = evaluate_time[r] print("active cost time:", datetime.now() - startTime) tagDict = dict(list(df.groupby('pr_number'))) """对已经有的特征向量和标签做训练集的拆分""" train_data = df.loc[df['label'] == False].copy(deep=True) test_data = df.loc[df['label']].copy(deep=True) train_data.drop(columns=['label'], inplace=True) test_data.drop(columns=['label'], inplace=True) """问题转化为多标签问题 train_data_y [{pull_number:[r1, r2, ...]}, ... ,{}] """ train_data_y = {} pull_number_list = train_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in pull_number_list: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) train_data_y[pull_number] = reviewers train_data.drop(columns=['review_user_login'], inplace=True) train_data.drop_duplicates(inplace=True) train_data.drop_duplicates(subset=['pr_number'], inplace=True) """训练集 结果做出多标签分类通用的模式""" train_data_y = DataProcessUtils.convertLabelListToDataFrame(train_data_y, pull_number_list, candicateNum) test_data_y = {} pull_number_list = test_data.drop_duplicates(['pr_number']).copy(deep=True)['pr_number'] for pull_number in test_data.drop_duplicates(['pr_number'])['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) test_data_y[pull_number] = reviewers test_data.drop(columns=['review_user_login'], inplace=True) test_data.drop_duplicates(inplace=True) test_data.drop_duplicates(subset=['pr_number'], inplace=True) # test_data_y = DataProcessUtils.convertLabelListToDataFrame(test_data_y, pull_number_list, candicateNum) test_data_y = DataProcessUtils.convertLabelListToListArray(test_data_y, pull_number_list) """获得pr list""" prList = list(test_data['pr_number']) """去除pr number""" test_data.drop(columns=['pr_number'], inplace=True) train_data.drop(columns=['pr_number'], inplace=True) test_data.drop(columns=['pr_created_at', 'pr_user_login', 'comment_at', 'label_y', 'label_m', 'response_time'], inplace=True) train_data.drop(columns=['pr_created_at', 'pr_user_login', 'comment_at', 'label_y', 'label_m', 'response_time'], inplace=True) """参数规范化""" if isSTD: stdsc = StandardScaler() train_data_std = stdsc.fit_transform(train_data) test_data_std = stdsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList elif isNOR: maxminsc = MinMaxScaler() train_data_std = maxminsc.fit_transform(train_data) test_data_std = maxminsc.transform(test_data) return train_data_std, train_data_y, test_data_std, test_data_y, convertDict, prList else: return train_data, train_data_y, test_data, test_data_y, convertDict, prList
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" """空comment的review包含na信息,但作为结果集是有用的,所以只对训练集去掉na""" # """处理NAN""" # df.dropna(how='any', inplace=True) # df.reset_index(drop=True, inplace=True) df['pr_title'].fillna(value='', inplace=True) df['pr_body'].fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['pr_author', 'reviewer']) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 textList = [] for row in df.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """初步尝试提取词干效果反而下降了 。。。。""" """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) """pull request的body""" pr_body = getattr(row, 'pr_body') pr_body_word_list = [x for x in FleshReadableUtils.word_list(pr_body) if x not in stopwords] """对单词做提取词干""" pr_body_word_list = nltkFunction.stemList(pr_body_word_list) tempList.extend(pr_body_word_list) textList.append(tempList) print(textList.__len__()) """对分词列表建立字典 并提取特征数""" dictionary = corpora.Dictionary(textList) print('词典:', dictionary) feature_cnt = len(dictionary.token2id) print("词典特征数:", feature_cnt) """根据词典建立语料库""" corpus = [dictionary.doc2bow(text) for text in textList] # print('语料库:', corpus) """语料库训练TF-IDF模型""" tfidf = models.TfidfModel(corpus) """再次遍历数据,形成向量,向量是稀疏矩阵的形式""" wordVectors = [] for i in range(0, df.shape[0]): wordVectors.append(dict(tfidf[dictionary.doc2bow(textList[i])])) """对已经有的本文特征向量和标签做训练集和测试集的拆分""" trainData_index = df.loc[df['label'] == False].index testData_index = df.loc[df['label'] == True].index """训练集""" train_data = [wordVectors[x] for x in trainData_index] """测试集""" test_data = [wordVectors[x] for x in testData_index] """填充为向量""" train_v_data = DataProcessUtils.convertFeatureDictToDataFrame(train_data, featureNum=feature_cnt) test_v_data = DataProcessUtils.convertFeatureDictToDataFrame(test_data, featureNum=feature_cnt) train_data = df.loc[df['label'] == False] train_data.reset_index(drop=True, inplace=True) test_data = df.loc[df['label'] == True] test_data.reset_index(drop=True, inplace=True) train_data = train_data.join(train_v_data) train_data.drop(columns=['label'], inplace=True) test_data = test_data.join(test_v_data) test_data.drop(columns=['label'], inplace=True) """8ii处理NAN""" train_data.dropna(how='any', inplace=True) train_data.reset_index(drop=True, inplace=True) train_data.fillna(value='', inplace=True) """先对tag做拆分""" trainDict = dict(list(train_data.groupby('pull_number'))) testDict = dict(list(test_data.groupby('pull_number'))) """过滤掉评论时间在数据集时间范围内之后的数据""" end_time = str(dates[2]) + "-" + str(dates[3]) + "-" + "01 00:00:00" train_data = train_data[train_data['commented_at'] < end_time] train_data.reset_index(drop=True, inplace=True) test_data_y = {} for pull_number in test_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(testDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) test_data_y[pull_number] = reviewers train_data_y = {} for pull_number in train_data.drop_duplicates(['pull_number'])['pull_number']: reviewers = list(trainDict[pull_number].drop_duplicates(['reviewer'])['reviewer']) train_data_y[pull_number] = reviewers return train_data, train_data_y, test_data, test_data_y, convertDict
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:作为测试的年月四元组 """ """注意: 输入文件中已经带有列名了""" """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """频率统计每一个reviewer的次数,排除数量过少的reviewer""" freq = {} for data in df.itertuples(index=False): name = data[list(df.columns).index('review_user_login')] if freq.get(name, None) is None: freq[name] = 0 """训练集用户次数加一 测试集直接保留 """ if not data[list(df.columns).index('label')]: freq[name] += 1 else: freq[name] += 1 num = 5 df['freq'] = df['review_user_login'].apply(lambda x: freq[x]) df = df.loc[df['freq'] > num].copy(deep=True) df.drop(columns=['freq'], inplace=True) df.reset_index(drop=True, inplace=True) print("after lifter unexperienced user:"******"""先对输入数据做精简 只留下感兴趣的数据""" df = df[['pull_number', 'review_user_login', 'commit_sha', 'file_filename', 'label']].copy(deep=True) print("before filter:", df.shape) df.drop_duplicates(inplace=True) print("after filter:", df.shape) """对人名字做数字处理""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) reviewer_num = convertDict.items().__len__() print("reviewer num:", convertDict.items().__len__()) """测试集 训练集做拆分""" train_raw_df = df.loc[df['label'] == 0].copy(deep=True) # userDict = dict(list(train_raw_df.groupby('review_user_login'))) # print(userDict) """建立用户历史review记录的TF-IDF模型""" """获取filepath -> sub_filepath映射表""" file_path_list = set(df['file_filename'].copy(deep=True)) file_path_dict = {} for file_path in file_path_list: # sub_file_path = splitFileName(file_path) sub_file_path = file_path.split('/') if file_path not in file_path_dict: file_path_dict[file_path] = set() file_path_dict[file_path] = file_path_dict[file_path].union(sub_file_path) """获取pr_number -> sub_filepath语料""" reviewer_to_file_path = df[['review_user_login', 'file_filename']] # 按照reviewer分组,获得原始语料(未经过分词的filepath)""" groups = dict(list(reviewer_to_file_path.groupby('review_user_login'))) # 获取目标语料(即经过自定义分词后的语料) reviewer_file_path_corpora = [] for reviewer in groups: paths = list(groups[reviewer]['file_filename']) sub_paths = list(map(lambda x: list(file_path_dict[x]), paths)) sub_paths = reduce(lambda x, y: x + y, sub_paths) reviewer_file_path_corpora.append(sub_paths) """计算tf-idf""" print("start tf_idf algorithm......") # 建立词典 dictionary = corpora.Dictionary(reviewer_file_path_corpora) # 基于词典建立新的语料库 corpus = [dictionary.doc2bow(text) for text in reviewer_file_path_corpora] # 用语料库训练TF-IDF模型 tf_idf_model = models.TfidfModel(corpus) # 得到加权矩阵 path_tf_tdf = list(tf_idf_model[corpus]) print(path_tf_tdf) """处理path_tf_tdf,构造pr_path加权矩阵""" print("start merge tf_idf to origin_df......") reviewer_list = list(groups.keys()) columns = ['review_user_login'] path_ids = list(dictionary.token2id.values()) path_ids = list(map(lambda x: str(x), path_ids)) columns.extend(path_ids) reviewer_path_weight_df = pandas.DataFrame(columns=columns).fillna(value=0) for index, row in enumerate(path_tf_tdf): """用字典的方式填充dataframe""" new_row = {'review_user_login': reviewer_list[index]} row = list(map(lambda x: (str(x[0]), x[1]), row)) path_weight = dict(row) new_row = dict(new_row, **path_weight) reviewer_path_weight_df = reviewer_path_weight_df.append(new_row, ignore_index=True) reviewer_path_weight_df = reviewer_path_weight_df.fillna(value=0) print(reviewer_path_weight_df.shape) """去除用户名""" reviewer_path_weight_df.drop(columns=['review_user_login'], axis=1, inplace=True) print("before pca size:", reviewer_path_weight_df.shape) """PCA""" pca = PCA(n_components=0.95) reviewer_path_weight_df = pca.fit_transform(reviewer_path_weight_df) print("after pca size:", reviewer_path_weight_df.shape) """特征标准归一化""" stdsc = StandardScaler() reviewer_path_weight_df_std = stdsc.fit_transform(reviewer_path_weight_df) M = [] N = [] max_cluster = min(20, reviewer_num) for n in range(2, 20): y_pred = KMeans(n_clusters=n, random_state=9).fit_predict(reviewer_path_weight_df_std) from sklearn import metrics # m = metrics.calinski_harabasz_score(reviewer_path_weight_df, y_pred) m = sklearn.metrics.silhouette_score(reviewer_path_weight_df_std, y_pred, sample_size=len(reviewer_path_weight_df_std), metric='euclidean') M.append(m) N.append(n) print(n ,m) print(y_pred) """聚类归属数量子图绘制""" nums = [] x = [] for i in range(0, n): nums.append(list(y_pred).count(i)) x.append(i) print(x) print(nums) plt.subplot(5, 4, n-1) plt.bar(x=range(0, n), height=nums) # plt.title('cluster%d'%n) plt.savefig(f'项目{projectName}聚类分布.png') # plt.show() M = DataFrame(M) N = DataFrame(N) # x_major_locator = MultipleLocator(1) # x轴间距为1 fig = plt.figure() # ax1 = fig.add_subplot(111) # ax1.scatter(N, M, s=40, marker='o') # ax1.xaxis.set_major_locator(x_major_locator) plt.plot(N, M, marker='o', markersize=5) plt.xlabel('cluster number') plt.ylabel('slihouette score') plt.xlim(-0.5, 22) # for a, b in zip(list(N), list(M)): # plt.text(a, b, b, ha='center', va='bottom', fontsize=20) # plt.legend() plt.savefig(f'项目{projectName}轮廓系数.png')
def preProcess(df, dates): """参数说明 df:读取的dataframe对象 dates:四元组,后两位作为测试的年月 (,,year,month) """ """注意: 输入文件中已经带有列名了""" t1 = datetime.now() """处理NAN""" df.dropna(how='any', inplace=True) df.reset_index(drop=True, inplace=True) df.fillna(value='', inplace=True) """对df添加一列标识训练集和测试集""" df['label'] = df['pr_created_at'].apply( lambda x: (time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_year == dates[2] and time.strptime(x, "%Y-%m-%d %H:%M:%S").tm_mon == dates[3])) """对reviewer名字数字化处理 存储人名映射字典做返回""" convertDict = DataProcessUtils.changeStringToNumber(df, ['review_user_login']) """先对tag做拆分""" tagDict = dict(list(df.groupby('pr_number'))) commentDf = df[['pr_number', 'review_user_login', 'comment_body', 'label']].copy(deep=True) """用于收集所有文本向量分词""" stopwords = SplitWordHelper().getEnglishStopList() # 获取通用英语停用词 """先尝试所有信息团在一起""" df = df[['pr_number', 'pr_title', 'pr_body', 'label']].copy(deep=True) df.drop_duplicates(inplace=True) df.reset_index(drop=True, inplace=True) """训练和测试做分割""" df_train = df.loc[df['label'] == 0].copy(deep=True) df_test = df.loc[df['label'] == 1].copy(deep=True) df_test.reset_index(drop=True, inplace=True) """收集训练集中的pr的文本作为 文档做LDA提取主题""" trainTextList = [] testTextList = [] for row in df.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的number""" pr_num = getattr(row, 'pr_number') label = getattr(row, 'label') """获取pull request的标题""" pr_title = getattr(row, 'pr_title') pr_title_word_list = [x for x in FleshReadableUtils.word_list(pr_title) if x not in stopwords] """对单词做提取词干""" pr_title_word_list = nltkFunction.stemList(pr_title_word_list) tempList.extend(pr_title_word_list) """pull request的body""" pr_body = getattr(row, 'pr_body') pr_body_word_list = [x for x in FleshReadableUtils.word_list(pr_body) if x not in stopwords] """对单词做提取词干""" pr_body_word_list = nltkFunction.stemList(pr_body_word_list) tempList.extend(pr_body_word_list) if label == 0: trainTextList.append(tempList) elif label == 1: testTextList.append(tempList) """收集 训练集中的comment""" trainCommentList = [] review_comment_map = {} # pr -> [(reviewer, [w1, w2, w3]), .....] for row in commentDf.itertuples(index=False, name='Pandas'): tempList = [] """获取pull request的number""" pr_num = getattr(row, 'pr_number') label = getattr(row, 'label') reviewer = getattr(row, 'review_user_login') """获取pull request的标题""" comment_body = getattr(row, 'comment_body') comment_body_word_list = [x for x in FleshReadableUtils.word_list(comment_body) if x not in stopwords] """对单词做提取词干""" comment_body_word_list = nltkFunction.stemList(comment_body_word_list) tempList.extend(comment_body_word_list) if review_comment_map.get(pr_num, None) is None: review_comment_map[pr_num] = [] if label == 0: review_comment_map[pr_num].append((reviewer, tempList.copy())) trainCommentList.append(tempList) """建立LDA模型提取数据""" # 接下来就是模型构建的步骤了,首先构建词频矩阵 allTextList = [] allTextList.extend(trainTextList) allTextList.extend(trainCommentList) dictionary = corpora.Dictionary(trainTextList) corpus = [dictionary.doc2bow(text) for text in trainTextList] lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20) topic_list = lda.print_topics(20) print("20个主题的单词分布为:\n") for topic in topic_list: print(topic) """建立训练集和测试集所需的主题分布 pr_num -> {[(t1, p1), (t2, p2), .....]} """ train_data = {} test_data = {} for index, d in enumerate(lda.get_document_topics([dictionary.doc2bow(text) for text in trainTextList])): train_data[df_train['pr_number'][index]] = d for index, d in enumerate(lda.get_document_topics([dictionary.doc2bow(text) for text in testTextList])): test_data[df_test['pr_number'][index]] = d train_data_y = {} # pr -> [(reviewer, [(comment1), (comment2) ...])] for pull_number in df.loc[df['label'] == False]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) reviewerList = [] for reviewer in reviewers: commentTopicList = [] for r, words in review_comment_map[pull_number]: if r == reviewer: commentTopicList.append(words) commentTopicList = lda.get_document_topics([dictionary.doc2bow(text) for text in commentTopicList]) reviewerList.append((reviewer, [x for x in commentTopicList])) train_data_y[pull_number] = reviewerList test_data_y = {} for pull_number in df.loc[df['label'] == True]['pr_number']: reviewers = list(tagDict[pull_number].drop_duplicates(['review_user_login'])['review_user_login']) reviewerList = [] for reviewer in reviewers: commentTopicList = [] for r, words in review_comment_map[pull_number]: if r == reviewer: commentTopicList.append(words) commentTopicList = lda.get_document_topics([dictionary.doc2bow(text) for text in commentTopicList]) reviewerList.append((reviewer, commentTopicList)) test_data_y[pull_number] = reviewerList print("preprocess cost time:", datetime.now() - t1) return train_data, train_data_y, test_data, test_data_y, convertDict