def get_issue_events_repo(gith, repo, project_name, start_date,
active_users): #Why Not get_events()?
path = super_path + '/' + project_name + '/Other_Activities'
os.makedirs(path, exist_ok=True)
exception_thrown = True
while (exception_thrown):
exception_thrown = False
if 'issues_events_repo.csv' in os.listdir(path):
issues_events_data = pandas.read_csv(path +
'/issues_events_repo.csv',
sep=',')
else:
issues_events_data = pandas.DataFrame(
columns=['id', 'date', 'event', 'creator_login'])
if 'issues_events_extraction.log' in os.listdir(path):
last_issues_page, last_issue, last_events_page = get_last_page_read(
path + '/issues_events_extraction.log')
else:
last_issues_page = 0
last_issue = ''
last_events_page = 0
if 'events_extraction_completed_issues.csv' in os.listdir(path):
completed_issues = pandas.read_csv(
path + '/events_extraction_completed_issues.csv', sep=',')
else:
completed_issues = pandas.DataFrame(columns=['id'])
logger = open(path + '/issues_events_extraction.log', 'a+')
### Get Other Issues Events
try:
issues_page = last_issues_page
issue_id = ''
page = 0
issues = repo.get_issues(state='all',
sort='created_at',
since=start_date)
num_issues = issues.totalCount
final_issues_page = int(num_issues / cfg.items_per_page)
for issues_page in range(last_issues_page, final_issues_page + 1):
cfg.waitRateLimit(gith)
current_issues_page = issues.get_page(issues_page)
for issue in current_issues_page:
cfg.waitRateLimit(gith)
issue_id = issue.id
if (issue_id not in completed_issues.id.tolist()):
if (issue_id != last_issue):
last_page = 0
else:
last_page = last_events_page
cfg.waitRateLimit(gith)
issue_events = issue.get_events()
num_items = issue_events.totalCount
final_page = int(num_items / cfg.items_per_page)
for page in range(last_page, final_page + 1):
cfg.waitRateLimit(gith)
issues_events_page = issue_events.get_page(page)
for event in issues_events_page:
cfg.waitRateLimit(gith)
event_id = event.id
if (event_id
not in issues_events_data.id.tolist()):
if (event.actor):
cfg.waitRateLimit(gith)
actor_login = event.actor.login
if (actor_login in active_users):
cfg.waitRateLimit(gith)
util.add(issues_events_data, [
event_id, event.created_at,
event.event, actor_login
])
util.add(completed_issues, issue_id)
if (len(issues_events_data) > 0):
issues_events_data.to_csv(path + '/issues_events_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_issues.to_csv(
path + '/events_extraction_completed_issues.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
os.rename(path + '/issues_events_repo.csv',
path + '/complete_issues_events_repo.csv')
print('{}: Issues Events Extraction Complete'.format(repo))
except github.GithubException:
print('Exception Occurred While Getting ISSUES EVENTS: Github')
logger.write(
'last_issues_page:{},last_issue:{},last_event_page:{}\n'.
format(issues_page, issue_id, page))
logger.flush()
if (len(issues_events_data) > 0):
issues_events_data.to_csv(path + '/issues_events_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_issues.to_csv(
path + '/events_extraction_completed_issues.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
exception_thrown = True
pass
except requests.exceptions.Timeout:
print('Exception Occurred While Getting ISSUES EVENTS: Timeout')
logger.write(
'last_issues_page:{},last_issue:{},last_event_page:{}\n'.
format(issues_page, issue_id, page))
logger.flush()
if (len(issues_events_data) > 0):
issues_events_data.to_csv(path + '/issues_events_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_issues.to_csv(
path + '/events_extraction_completed_issues.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
exception_thrown = True
pass
except:
print('Execution Interrupted While Getting ISSUES EVENTS')
logger.write(
'last_issues_page:{},last_issue:{},last_event_page:{}\n'.
format(issues_page, issue_id, page))
logger.flush()
if (len(issues_events_data) > 0):
issues_events_data.to_csv(path + '/issues_events_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_issues.to_csv(
path + '/events_extraction_completed_issues.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
raise
def get_issues_prs(gith, repo, project_name, start_date, developer_login):
path = super_path + '/' + project_name + '/Other_Activities/' + developer_login
os.makedirs(path, exist_ok=True)
exception_thrown = True
while (exception_thrown):
exception_thrown = False
logger = open(path + '/issues_pr_extraction.log', 'a+')
### Get Issue / Pull Requests
created_issues_prs = repo.get_issues(state='all',
sort='created_at',
since=start_date,
creator=developer_login)
count_exception = True
while (count_exception):
count_exception = False
try:
num_items = created_issues_prs.totalCount
except github.GithubException:
print(
'Failed to get ISSUES/PRs Number from User {} and Project {} (TIMEOUT: Retrying)'
.format(developer_login, project_name))
count_exception = True
pass
except requests.exceptions.Timeout:
print(
'Failed to get ISSUES/PRs Number from User {} and Project {} (TIMEOUT: Retrying)'
.format(developer_login, project_name))
count_exception = True
pass
except:
print(
'Failed to get ISSUES/PRs Number from User {} and Project {} (Probably Empty)'
.format(developer_login, project_name))
return
last_page = int(num_items / cfg.items_per_page)
last_page_read = 0
if 'issues_pr_creation.csv' in os.listdir(path):
issues_prs_data = pandas.read_csv(path + '/issues_pr_creation.csv',
sep=',')
last_page_read = get_last_page_read_short(
path + '/issues_pr_extraction.log')
else:
issues_prs_data = pandas.DataFrame(
columns=['id', 'date', 'creator_login'])
try:
for page in range(last_page_read, last_page + 1):
created_issues_prs_page = created_issues_prs.get_page(page)
for issue in created_issues_prs_page:
issue_id = issue.id
if (issue_id not in issues_prs_data.id.tolist()):
if (issue.user):
cfg.waitRateLimit(gith)
util.add(
issues_prs_data,
[issue_id, issue.created_at, issue.user.login])
logger.write('last_page_read:{}\n'.format(page))
logger.flush()
if (len(issues_prs_data) > 0):
issues_prs_data.to_csv(path + '/issues_pr_creation.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
print(
'{}: Issues/Pulls Extraction Complete'.format(developer_login))
except github.GithubException:
print('Exception Occurred While Getting ISSUES/PULLS: Github')
logger.write('last_page_read:{}\n'.format(page))
logger.flush()
if (len(issues_prs_data) > 0):
issues_prs_data.to_csv(path + '/issues_pr_creation.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
#get_issues_prs(gith, repo, project_name, start_date, developer_login)
exception_thrown = True
pass
except requests.exceptions.Timeout:
print('Exception Occurred While Getting ISSUES/PULLS: Timeout')
logger.write('last_page_read:{}\n'.format(page))
logger.flush()
if (len(issues_prs_data) > 0):
issues_prs_data.to_csv(path + '/issues_pr_creation.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
#get_issues_prs(gith, repo, project_name, start_date, developer_login)
exception_thrown = True
pass
except:
print('Execution Interrupted While Getting ISSUES/PULLS')
logger.write('last_page_read:{}\n'.format(page))
logger.flush()
if (len(issues_prs_data) > 0):
issues_prs_data.to_csv(path + '/issues_pr_creation.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
raise
def writeCommitFile_Login(gith, project_url, start_date, end_date, path):
import os, requests
logger = open(path + '/commits_extraction.log', 'a+')
exception_thrown = True
while (exception_thrown):
exception_thrown = False
cfg.waitRateLimit(gith)
repo = gith.get_repo(project_url)
commits = repo.get_commits(
since=start_date, until=end_date
) #Fake users to be filtered out (author_id NOT IN (SELECT id from users where fake=1))
try:
num_items = commits.totalCount
except:
print(
'Failed to get commits from this project (Probably Empty): ' +
project_url)
return
last_page = int(num_items / cfg.items_per_page)
last_page_read = 0
if 'commits_raw_login.csv' in os.listdir(path):
commits_data = pandas.read_csv(path + '/commits_raw_login.csv',
sep=',')
last_page_read = util.get_last_page_read_short(
path + '/commits_extraction.log')
else:
commits_data = pandas.DataFrame(
columns=['sha', 'author_id', 'date'])
if 'excluded_for_NoneType.csv' in os.listdir(path):
excluded_commits = pandas.read_csv(path +
'/excluded_for_NoneType.csv',
sep=',')
else:
excluded_commits = pandas.DataFrame(columns=['sha'])
try:
for page in range(last_page_read, last_page + 1):
commits_page = commits.get_page(page)
for commit in commits_page:
cfg.waitRateLimit(gith)
sha = commit.sha
if ((sha not in commits_data.sha.tolist()) &
(sha not in excluded_commits.sha.tolist())):
if (
commit.author
): ### If author is NoneType, that means the author is no longer active in GitHub
cfg.waitRateLimit(gith)
author_id = commit.author.login ### HERE IS THE DIFFERENCE
date = commit.commit.author.date
util.add(commits_data, [sha, author_id, date])
if (len(commits_data) > 0):
commits_data.to_csv(path + '/commits_raw_login.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
except github.GithubException:
print('Exception Occurred While Getting COMMITS: Github')
if (len(commits_data) > 0):
commits_data.to_csv(path + '/commits_raw_login.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
logger.write('last_page:{}\n'.format(page))
logger.flush()
exception_thrown = True
pass
except requests.exceptions.Timeout:
print('Exception Occurred While Getting COMMITS: Timeout')
if (len(commits_data) > 0):
commits_data.to_csv(path + '/commits_raw_login.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
logger.write('last_page:{}\n'.format(page))
logger.flush()
exception_thrown = True
pass
except AttributeError:
print(
'Exception Occurred While Getting COMMIT DATA: NoneType for Author. SHA: '
+ sha)
util.add(excluded_commits, [sha])
if (len(commits_data) > 0):
commits_data.to_csv(path + '/commits_raw_login.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
excluded_commits.to_csv(path + '/excluded_for_NoneType.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
logger.write('last_page:{}\n'.format(page))
logger.flush()
exception_thrown = True
except:
print('Execution Interrupted While Getting COMMITS')
if (len(commits_data) > 0):
commits_data.to_csv(path + '/commits_raw_login.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
logger.write('last_page:{}\n'.format(page))
logger.flush()
raise
def get_pulls_comments_repo(gith, repo, project_name, start_date,
active_users):
path = super_path + '/' + project_name + '/Other_Activities'
os.makedirs(path, exist_ok=True)
exception_thrown = True
while (exception_thrown):
exception_thrown = False
if 'pulls_comments_repo.csv' in os.listdir(path):
pulls_comments_data = pandas.read_csv(path +
'/pulls_comments_repo.csv',
sep=',')
else:
pulls_comments_data = pandas.DataFrame(
columns=['id', 'date', 'creator_login'])
if 'pulls_comments_extraction.log' in os.listdir(path):
last_pulls_page, last_pull, last_comments_page = get_last_page_read(
path + '/pulls_comments_extraction.log')
else:
last_pulls_page = 0
last_pull = ''
last_comments_page = 0
if 'comments_extraction_completed_pulls.csv' in os.listdir(path):
completed_pulls = pandas.read_csv(
path + '/comments_extraction_completed_pulls.csv', sep=',')
else:
completed_pulls = pandas.DataFrame(columns=['id'])
logger = open(path + '/pulls_comments_extraction.log', 'a+')
### Get Comments on Pull
try:
pulls_page = last_pulls_page
pull_id = ''
page = 0
pulls = repo.get_pulls(state='all', sort='created_at')
num_pulls = pulls.totalCount
final_pulls_page = int(num_pulls / cfg.items_per_page)
for pulls_page in range(last_pulls_page, final_pulls_page + 1):
cfg.waitRateLimit(gith)
current_pulls_page = pulls.get_page(pulls_page)
for pull in current_pulls_page:
cfg.waitRateLimit(gith)
pull_id = pull.id
if (pull_id not in completed_pulls.id.tolist()):
if (pull_id != last_pull):
last_page = 0
else:
last_page = last_comments_page
cfg.waitRateLimit(gith)
pulls_comments = pull.get_comments()
num_items = pulls_comments.totalCount
final_page = int(num_items / cfg.items_per_page)
for page in range(last_page, final_page + 1):
cfg.waitRateLimit(gith)
pulls_comments_page = pulls_comments.get_page(page)
for comment in pulls_comments_page:
cfg.waitRateLimit(gith)
comment_id = comment.id
if (comment_id not in
pulls_comments_data.id.tolist()):
if (comment.user):
cfg.waitRateLimit(gith)
user_login = comment.user.login
if (user_login in active_users):
cfg.waitRateLimit(gith)
util.add(pulls_comments_data, [
comment_id, comment.created_at,
user_login
])
util.add(completed_pulls, pull_id)
if (len(pulls_comments_data) > 0):
pulls_comments_data.to_csv(path + '/pulls_comments_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_pulls.to_csv(
path + '/comments_extraction_completed_pulls.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
os.rename(path + '/pulls_comments_repo.csv',
path + '/complete_pulls_comments_repo.csv')
print('{}: Pulls Comments Extraction Complete'.format(repo))
except github.GithubException:
print('Exception Occurred While Getting PULLS COMMENTS: Github')
logger.write(
'last_pulls_page:{},last_pull:{},last_comment_page:{}\n'.
format(pulls_page, pull_id, page))
logger.flush()
if (len(pulls_comments_data) > 0):
pulls_comments_data.to_csv(path + '/pulls_comments_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_pulls.to_csv(
path + '/comments_extraction_completed_pulls.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
exception_thrown = True
pass
except requests.exceptions.Timeout:
print('Exception Occurred While Getting PULLS COMMENTS: Timeout')
logger.write(
'last_pulls_page:{},last_pull:{},last_comment_page:{}\n'.
format(pulls_page, pull_id, page))
logger.flush()
if (len(pulls_comments_data) > 0):
pulls_comments_data.to_csv(path + '/pulls_comments_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_pulls.to_csv(
path + '/comments_extraction_completed_pulls.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
exception_thrown = True
pass
except:
print('Execution Interrupted While Getting PULLS COMMENTS')
logger.write(
'last_pulls_page:{},last_pull:{},last_comment_page:{}\n'.
format(pulls_page, pull_id, page))
logger.flush()
if (len(pulls_comments_data) > 0):
pulls_comments_data.to_csv(path + '/pulls_comments_repo.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
completed_pulls.to_csv(
path + '/comments_extraction_completed_pulls.csv',
sep=',',
na_rep='NA',
header=True,
index=False,
mode='w',
encoding='utf-8',
quoting=None,
quotechar='"',
line_terminator='\n',
decimal='.')
raise
tm = cfg.TokenManagement.getInstance()
g = Github(tm.newToken())
g.per_page = cfg.items_per_page
p_names = cfg.p_names
p_urls = cfg.p_urls
super_path = cfg.super_path
for i in range(0, len(p_urls)):
chosen_project = i # FROM 0 TO n-1
project_name = p_names[chosen_project]
partial_project_url = p_urls[chosen_project]
cfg.waitRateLimit(g)
repo = g.get_repo(partial_project_url)
logging.basicConfig(filename=super_path + '/CommitExtraction.log',
level=logging.INFO)
logging.info('Project: ' + project_name + ' PID: ' + project_name +
' Started')
cfg.waitRateLimit(g)
project_start_dt = repo.created_at
project_start = project_start_dt.strftime('%Y-%m-%d %H:%M:%S')
path = (super_path + '/' + project_name)
os.makedirs(path, exist_ok=True)
collection_day = datetime.strptime(cfg.collection_date, '%Y-%m-%d')