def get_repo_PRlist(repo, type, renew):
api = GitHubAPI()
save_path = LOCAL_DATA_PATH + '/pr_data/' + repo + '/%s_list.json' % type
# todo: could be extended to analyze forks in the future
if type == 'fork':
save_path = LOCAL_DATA_PATH + '/result/' + repo + '/forks_list.json'
if (os.path.exists(save_path)) and (not renew):
print("read from local files and return")
try:
return localfile.get_file(save_path)
except:
pass
print('files does not exist in local disk, start to fetch new list for ',
repo, type)
if (type == 'pull') or (type == 'issue'):
ret = api.request('repos/%s/%ss' % (repo, type),
state='all',
paginate=True)
else:
if type == 'branch':
type = 'branche'
ret = api.request('repos/%s/%ss' % (repo, type), True)
localfile.write_to_file(save_path, ret)
return ret
def fetch_file_list(pull, renew=False):
repo, num = pull["base"]["repo"]["full_name"], str(pull["number"])
save_path = LOCAL_DATA_PATH + '/pr_data/' + repo + '/' + num + '/raw_diff.json'
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
# t = api.get('repos/%s/pulls/%s/files?page=3' % (repo, num))
t = api.request('repos/%s/pulls/%s/files?page=3' % (repo, num))
file_list = []
if len(t) > 0:
raise Exception('too big', pull['html_url'])
else:
li = api.request('repos/%s/pulls/%s/files' % (repo, num),
paginate=True)
# li = api.request( 'repos/%s/pulls/%s/files' % (repo, num), True)
time.sleep(0.8)
for f in li:
if f.get('changes', 0) <= 5000 and ('filename' in f) and ('patch'
in f):
file_list.append(parse_diff(f['filename'], f['patch']))
localfile.write_to_file(save_path, file_list)
return file_list
def filterNonCodeFiles(file_list, outfile_prefix):
newFileList = []
count = 0
for f in file_list:
if count > 500:
localfile.write_to_file(outfile_prefix + "/toobig.txt", '500file')
return []
if not language_tool.is_text(f['name']):
newFileList.append(f)
count += 1
return newFileList
def get_PR(repo, num, renew=False):
save_path = LOCAL_DATA_PATH + '/pr_data/%s/%s/api.json' % (repo, num)
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
r = api.request('repos/%s/pulls/%s' % (repo, num))
time.sleep(3.0)
localfile.write_to_file(save_path, r)
return r
def get_pull_commit(pull, renew=False):
save_path = LOCAL_DATA_PATH + '/pr_data/%s/%s/commits.json' % (
pull["base"]["repo"]["full_name"], pull["number"])
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
commits = api.request(pull['commits_url'].replace(
'https://api.github.com/', ''),
paginate=True,
state='all')
time.sleep(0.7)
localfile.write_to_file(save_path, commits)
return commits
def get_pr_commit(repo, pr_id, renew=False):
save_path = LOCAL_DATA_PATH + '/pr_data/%s/%s/commits.json' % (repo, pr_id)
commit_url = 'repos/%s/pulls/%s/commits' % (repo, pr_id)
if os.path.exists(save_path) and (not renew) and (
os.stat(save_path).st_size > 2):
try:
return localfile.get_file(save_path)
except:
pass
# commits = api.request(pull['commits_url'].replace('https://api.github.com/', ''), True)
api = GitHubAPI()
commits = api.request(commit_url.replace('https://api.github.com/', ''),
paginate=True,
state='all')
time.sleep(0.7)
localfile.write_to_file(save_path, commits)
return commits
def fetch_commit(url, renew=False):
api = GitHubAPI()
save_path = LOCAL_DATA_PATH + '/pr_data/%s.json' % url.replace(
'https://api.github.com/repos/', '')
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
c = api.request(url)
time.sleep(0.7)
file_list = []
for f in c['files']:
if 'patch' in f:
file_list.append(
fetch_raw_diff.parse_diff(f['filename'], f['patch']))
localfile.write_to_file(save_path, file_list)
return file_list
def fetch_file_list(repo, num, renew=False):
api = GitHubAPI()
# repo, num = pull["base"]["repo"]["full_name"], str(pull["number"])
outfile_prefix = init.local_pr_data_dir + repo + "/" + str(num)
save_path = outfile_prefix + '/raw_diff.json'
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
file_list = []
li = api.request('repos/%s/pulls/%s/files' % (repo, num), paginate=True)
time.sleep(0.8)
for f in li:
if f.get('changes', 0) <= 5000 and ('filename' in f) and ('patch'
in f):
file_list.append(
fetch_raw_diff.parse_diff(f['filename'], f['patch']))
localfile.write_to_file(save_path, file_list)
return file_list
def get_another_pull(pull, renew=False):
api = GitHubAPI()
save_path = LOCAL_DATA_PATH + '/pr_data/%s/%s/another_pull.json' % (
pull["base"]["repo"]["full_name"], pull["number"])
if os.path.exists(save_path) and (not renew):
try:
return localfile.get_file(save_path)
except:
pass
comments_href = pull["_links"]["comments"][
"href"] # found cites in comments, but checking events is easier.
comments = api.request(comments_href, True)
time.sleep(0.7)
candidates = []
for comment in comments:
candidates.extend(get_pr_and_issue_numbers(comment["body"]))
candidates.extend(get_pr_and_issue_numbers(pull["body"]))
result = list(set(candidates))
localfile.write_to_file(save_path, result)
return result