def buildByList(repoList, corpusName): corpus = open('corpus/nocode%s.dat' % corpusName, "w") commitCorpus = open('corpus/commit%s.dat' % corpusName, "w") issueCorpus = open('corpus/issue%s.dat' % corpusName, "w") try: print 'start' for i in range(len(repoList)): repoId = repoList[i]['id'] repoPath = repoList[i]['path'] try: # commit part gitRe = gitResolver.GitResolver(repoPath) commits = gitRe.getCommits() print repoPath, ":", len(commits) for commit in commits: seqs = preprocessor.preprocessNoCamel( commit.message.decode('utf-8')) if len(seqs): # 不是空列表 for seq in seqs: for word in seq: corpus.write(word.encode('utf-8')) corpus.write(" ") commitCorpus.write(word.encode('utf-8')) commitCorpus.write(" ") corpus.write("\n") commitCorpus.write("\n") # issue part issues = mysqlOperator.selectAllIssueInOneRepo(repoId) print repoId, ":", len(issues) for issue in issues: titleSeqs = preprocessor.preprocessNoCamel( issue[4].decode('utf-8')) if len(titleSeqs): # 不是空列表 for titleSeq in titleSeqs: for word in titleSeq: corpus.write(word.encode('utf-8')) corpus.write(" ") issueCorpus.write(word.encode('utf-8')) issueCorpus.write(" ") corpus.write("\n") issueCorpus.write("\n") if issue[5]: body = preprocessor.processHTMLNoCamel( issue[5].decode('utf-8')) if len(body): # 不是空列表 for bodySeq in body: for word in bodySeq: corpus.write(word.encode('utf-8')) corpus.write(" ") issueCorpus.write(word.encode('utf-8')) issueCorpus.write(" ") corpus.write("\n") issueCorpus.write("\n") except BaseException, e: print "***", repoId, ":", e print traceback.format_exc() print 'end'
def buildIssuePart(): repos = linkOperator.selectRepoOver(5000) textCorpus = open('issueText.dat', "w") codeCorpus = open('issueCode.dat', "w") try: print 'start' for highRepo in repos: try: issues = mysqlOperator.selectAllIssueInOneRepo(highRepo[0]) print highRepo[0], ":", len(issues) for issue in issues: titleWords = preprocessor.preprocessToWord( issue[4].decode('utf-8')) if len(titleWords): # 不是空列表 for word in titleWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if issue[5]: body = preprocessor.processHTML( issue[5].decode('utf-8')) bodyWords = body[1] codeWords = body[0] if len(bodyWords): # 不是空列表 for word in bodyWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if len(codeWords): # 不是空列表 for word in codeWords: codeCorpus.write(word.encode('utf-8')) codeCorpus.write(" ") codeCorpus.write("\n") comments = mysqlOperator.selectCommentInOneIssue(issue[1]) for comment in comments: temp = preprocessor.processHTML( comment[4].decode('utf-8')) cBodyWords = temp[1] cCodeWords = temp[0] if len(cBodyWords): # 不是空列表 for word in cBodyWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if len(cCodeWords): # 不是空列表 for word in cCodeWords: codeCorpus.write(word.encode('utf-8')) codeCorpus.write(" ") codeCorpus.write("\n") except BaseException, e: print "***", highRepo[0], ":", e print traceback.format_exc() print 'end'
def buildLinks(repoId): print 'start' try: repoPath = nocodeRepoInfo.REPO_MAP[ nocodeRepoInfo.USE_REPO_INDEX]['path'] gitRepo = gitResolver.GitResolver(repoPath) issues = mysqlOperator.selectAllIssueInOneRepo(repoId) commits = gitRepo.getCommits() # repoName = re.sub(r'https://github.com/', '', repo[1], 0, re.I) print '==============', repoPath, 'Start' for commit in commits: commitSha = str(commit.hexsha.encode("utf-8")) print commitSha commitIssues = mysqlOperator.selectExistIssueOnCommit( (repoId, commitSha)) trueLinks = [] for ci in commitIssues: if ci[0] in trueLinks: pass else: trueLinks.append(ci[0]) linkOperator.insertLink( ('true_link_%d' % repoId, repoId, commitSha, ci[0])) for issue in issues: if isUnlabeled(issue, gitRepo.getDateTime(commit)): if len(commitIssues) > 0: if issue[1] in trueLinks: pass else: linkOperator.insertLink( ('false_link_%d' % repoId, repoId, commitSha, issue[1])) else: pass print '==============', repoPath, 'End' except Exception, e: print 'Error:', repoPath print traceback.format_exc()
def buildIssueAndCommitSeq(repoId, repoPath, corpusName): corpus = open('corpus/code%s.dat' % corpusName, "w") try: print 'start' try: # commit part gitRe = gitResolver.GitResolver(repoPath) commits = gitRe.getCommits() print repoPath, ":", len(commits) for commit in commits: diffs = gitRe.getOneDiff(commit) for diff in diffs: diffCode = preprocessor.processDiffCode(diff.diff) if len(diffCode): for word in diffCode: corpus.write(word.encode('utf-8')) corpus.write(" ") corpus.write("\n") # issue part issues = mysqlOperator.selectAllIssueInOneRepo(repoId) print repoId, ":", len(issues) for issue in issues: if issue[5]: bodycode = preprocessor.getIssueCode( issue[5].decode('utf-8')) if len(bodycode): # 不是空列表 for word in bodycode: corpus.write(word.encode('utf-8')) corpus.write(" ") corpus.write("\n") except BaseException, e: print "***", repoId, ":", e print traceback.format_exc() print 'end'
# -*- coding: UTF-8 -*- from database import linkOperator, mysqlOperator projects = mysqlOperator.selectAllHighRepository() print 'start' for repo in projects: issues = mysqlOperator.selectAllIssueInOneRepo(repo[0]) issueLen = len(issues) linkLen = 0 for issue in issues: links = mysqlOperator.selectTrueLinkInOneIssue(issue[1]) linkLen = linkLen + len(links) turple = (repo[0], repo[1], issueLen, linkLen) print turple, '\n' linkOperator.insertOneRepo(turple) print 'end' linkOperator.close() mysqlOperator.close()
def buildIssueAndCommit(): repos = linkOperator.selectOneRepo(50904245) # repos = linkOperator.selectRepoOver(5000) textCorpus = open('text50904245.dat', "w") codeCorpus = open('code50904245.dat', "w") try: print 'start' for highRepo in repos: try: # commit part path = getPath(highRepo[1]) gitRe = gitResolver.GitResolver(path) commits = gitRe.getCommits() print path, ":", len(commits) for commit in commits: words = preprocessor.preprocessToWord( commit.message.decode('utf-8')) if len(words): # 不是空列表 for word in words: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") diffs = gitRe.getOneDiff(commit) for diff in diffs: diffCode = preprocessor.processDiffCode(diff.diff) preDiffCode = preprocessor.processPreDiffCode( diff.diff) if len(diffCode): for code in diffCode: codeCorpus.write(code) codeCorpus.write(" ") codeCorpus.write("\n") if len(preDiffCode): for code in preDiffCode: codeCorpus.write(code) codeCorpus.write(" ") codeCorpus.write("\n") # issue part issues = mysqlOperator.selectAllIssueInOneRepo(highRepo[0]) print highRepo[0], ":", len(issues) for issue in issues: titleWords = preprocessor.preprocessToWord( issue[4].decode('utf-8')) if len(titleWords): # 不是空列表 for word in titleWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if issue[5]: body = preprocessor.processHTML( issue[5].decode('utf-8')) bodyWords = body[1] codeWords = body[0] if len(bodyWords): # 不是空列表 for word in bodyWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if len(codeWords): # 不是空列表 for word in codeWords: codeCorpus.write(word.encode('utf-8')) codeCorpus.write(" ") codeCorpus.write("\n") comments = mysqlOperator.selectCommentInOneIssue(issue[1]) for comment in comments: temp = preprocessor.processHTML( comment[4].decode('utf-8')) cBodyWords = temp[1] cCodeWords = temp[0] if len(cBodyWords): # 不是空列表 for word in cBodyWords: textCorpus.write(word.encode('utf-8')) textCorpus.write(" ") textCorpus.write("\n") if len(cCodeWords): # 不是空列表 for word in cCodeWords: codeCorpus.write(word.encode('utf-8')) codeCorpus.write(" ") codeCorpus.write("\n") except BaseException, e: print "***", highRepo[0], ":", e print traceback.format_exc() print 'end'