def retry_if_neccessary(gotten_tag, tagname, objectname, arg_objectname):
how_long = 60
if gotten_tag is None:
#retry 3 times
for i in range(0, 3):
time.sleep(how_long)
how_long *= 3
while True:
try:
local_filename_html, headers_html = urllib.urlretrieve(
html_addr, filename + '.html')
break
except IOError:
io_socket_message = 'Socket error while retrieving HTML' +\
' file from GitHub! Internet or ' +\
'GitHub down? Retry after 1 minute'
scream.ssay(io_socket_message)
scream.log_warning(io_socket_message)
time.sleep(60)
soup = BeautifulSoup(html_content_file)
gotten_tag = soup.find(tagname, {objectname: arg_objectname})
if gotten_tag is not None:
raise StopIteration
if gotten_tag is None:
#nothing to do here, lets move on
scream.ssay('orphaned' + filename + '.json')
scream.log_error(
filename + '.json' + 'is without proper html. ' +
'GitHub not responding or giving 404/501 erorr ??')
return None
scream.say(
'No action required. retry_if_neccessary() returning gotten_tag')
return gotten_tag
def retry_if_neccessary(gotten_tag, tagname, objectname, arg_objectname):
how_long = 60
if gotten_tag is None:
#retry 3 times
for i in range(0, 3):
time.sleep(how_long)
how_long *= 3
while True:
try:
local_filename_html, headers_html = urllib.urlretrieve(
html_addr, filename + '.html')
break
except IOError:
io_socket_message = 'Socket error while retrieving HTML' +\
' file from GitHub! Internet or ' +\
'GitHub down? Retry after 1 minute'
scream.ssay(io_socket_message)
scream.log_warning(io_socket_message)
time.sleep(60)
soup = BeautifulSoup(html_content_file)
gotten_tag = soup.find(tagname, {objectname: arg_objectname})
if gotten_tag is not None:
raise StopIteration
if gotten_tag is None:
#nothing to do here, lets move on
scream.ssay('orphaned' + filename + '.json')
scream.log_error(filename + '.json' + 'is without proper html. ' +
'GitHub not responding or giving 404/501 erorr ??')
return None
scream.say('No action required. retry_if_neccessary() returning gotten_tag')
return gotten_tag
def slice_queue(queue, begin_arg, end_arg, percentage=False):
scream.log_warning("Slicing the dequeue, percentage=" + str(percentage), True)
if percentage:
begin_arg = int( str( len(queue) * begin_arg ) )
end_arg = int( str( len(queue) * end_arg ) )
# percentage threshold
right_index = len(queue) - end_arg
for i in xrange(1, begin_arg):
queue.popleft()
for x in xrange(1, right_index):
queue.pop()
def execute_check():
parser = argparse.ArgumentParser()
parser.add_argument("-v",
"--verbose",
help="verbose messaging ? [True/False]",
action="store_true")
args = parser.parse_args()
if args.verbose:
scream.intelliTag_verbose = True
scream.say("verbosity turned on")
threads = []
# init connection to database
first_conn = MSQL.connect(host=IP_ADDRESS,
port=3306,
user=open('mysqlu.dat', 'r').read(),
passwd=open('mysqlp.dat', 'r').read(),
db="github",
connect_timeout=50000000,
charset='utf8',
init_command='SET NAMES UTF8',
use_unicode=True)
print 'Testing MySql connection...'
print 'Pinging database: ' + (str(first_conn.ping(True)) if
first_conn.ping(True) is not None else 'NaN')
cursor = first_conn.cursor()
cursor.execute(
r'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = "%s"'
% 'github')
rows = cursor.fetchall()
print 'There are: ' + str(
rows[0][0]) + ' table objects in the local GHtorrent copy'
cursor.execute(
r'SELECT table_name FROM information_schema.tables WHERE table_schema = "%s"'
% 'github')
rows = cursor.fetchall()
if (u'users', ) and (u'projects', ) in rows:
print 'All neccesary tables are there.'
else:
print 'Your database does not fit a typical description of a GitHub Torrent copy..'
sys.exit(0)
sample_tb_name = raw_input(
"Please enter table/view name (of chosen data sample): ")
cursor.execute(r'select count(distinct name) from ' + str(sample_tb_name) +
' where ((name is not NULL) and (gender is NULL))')
rows = cursor.fetchall()
record_count = rows[0][0]
cursor.close()
scream.say(
"Database seems to be working. Move on to getting list of users.")
# populate list of users to memory
cursor = first_conn.cursor()
is_locked_tb = raw_input("Should I update [users] table instead of [" +
str(sample_tb_name) + "]? [y/n]: ")
is_locked_tb = True if is_locked_tb in ['yes', 'y'] else False
print 'Querying all names from the observations set.. This can take around 25-30 sec.'
cursor.execute(r'select distinct name from ' + str(sample_tb_name) +
' where ((name is not NULL) and (gender is NULL))')
# if you are interested in how this table was created, you will probably need to read our paper and contact us as well
# because we have some more tables with aggregated data compared to standard GitHub Torrent collection
row = cursor.fetchone()
iterator = 1.0
min_name_length = 2
print 'We hypothetize that minimum name length are ' \
+ str(min_name_length) + ' characters, like Ho, Sy, Lu'
# http://www.answers.com/Q/What_is_the_shortest_name_in_the_world
while row is not None:
fullname = unicode(row[0])
scream.log("\tFullname is: " + str(fullname.encode('unicode_escape')))
iterator += 1
print "[Progress]: " + str(
(iterator / record_count) *
100) + "% ----------- " # [names] size: " + str(len(names))
if len(fullname) < min_name_length:
scream.log_warning(
"--Found too short name field (" +
str(fullname.encode('utf-8')) + ") from DB. Skipping..", True)
row = cursor.fetchone()
continue
name = fullname.split()[0]
# I find it quite uncommon to seperate name from surname with something else than a space
# it does occur, but it's not in my interest to detect such human-generated dirty data at the moment
scream.log("\tName is: " + str(name.encode('unicode_escape')))
if name in names:
if fullname in names[name]['persons']:
scream.say(
"\tSuch fullname already classified! Rare, but can happen. Move on."
)
else:
scream.say(
"\tAdding fullname to already classified name. Move on")
names[name]['persons'].append(fullname)
else:
scream.say("\tNew name. Lets start classification.")
names[name] = {'persons': list(), 'classification': None}
names[name]['persons'].append(fullname)
scream.say("\tStart the worker on name: " +
str(name.encode('utf-8')) + " as deriven from: " +
str(fullname.encode('utf-8')))
# start the worker
gg = GeneralGetter(int(iterator), name)
scream.say('Creating instance of GeneralGetter complete')
scream.say('Appending thread to collection of threads')
threads.append(gg)
scream.say('Append complete, threads[] now have size: ' +
str(len(threads)))
scream.log_debug(
'Starting thread ' + str(int(iterator) - 1) + '....', True)
gg.start()
while (num_working(threads) > 3):
time.sleep(
0.2
) # sleeping for 200 ms - there are already 3 active threads..
row = cursor.fetchone()
cursor.close()
print "Finished getting gender data, moving to database update..."
for key in names.keys():
collection = names[key]
gender = collection['classification']
for fullname in names[key]['persons']:
cursor = first_conn.cursor()
update_query = r'UPDATE {2} SET gender = {0} where name = "{1}"'.format(
gender,
fullname.encode('utf-8').replace('"', '\\"'),
'users' if is_locked_tb else sample_tb_name)
print update_query
cursor.execute(update_query)
cursor.close()
first_conn.close()
})
if failed is not None:
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
gender_tag = local_soup.find(
"span",
{"id": "ctl00_ContentPlaceHolder1_" + "LabelGenderFound"})
if ((gender_tag is not None) and (gender_tag.contents is not None)
and (len(gender_tag.contents) > 0)):
gender = gender_tag.contents[0].string
scream.say(gender)
persist_users[s] = s + ',' + fullname.strip() + ',' + gender
return s + ',' + fullname.strip() + ',' + gender
else:
scream.log_warning(
'Something really wrong, on result page there ' +
'was no not-found label neither a proper result')
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
else:
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
else:
scream.say('Fullname not provided')
persist_users[s] = None
return s
# This method tries to get HTML discussion site again again
# when there is a problem with finding by th BS
# proper html tags which allways indicate author
branches = enumarables[1]
branches_number = analyze_tag(branches.find("span", {"class": "num"}))
print branches_number
releases = enumarables[2]
releases_number = analyze_tag(releases.find("span", {"class": "num"}))
print releases_number
contributors = enumarables[3]
contributors_number = analyze_tag(contributors.find("span", {"class": "num"}))
print contributors_number
break
except TypeError as ot:
if use_selenium:
output_nonexistent_writer.writerow(row)
break
# i already implemented selenium, so this won't have appliations anymore i believe :)
scream.log_warning("TypeError error({0})".format(ot))
scream.say(contributors)
scream.say('Timeout: 5s')
time.sleep(5)
scream.say('Will try again... GitHub didnt manage to fetch on time?')
except:
if use_selenium:
output_nonexistent_writer.writerow(row)
break
finally:
scream.say('Moving on to writing row and next record...')
row.append(commits_number)
row.append(branches_number)
row.append(releases_number)
row.append(contributors_number)
moredata_writer.writerow(row)
if not ((resume_on_repo_name == repo.getName()) and
(resume_on_repo_owner == repo.getOwner())):
iteration_step_count += 1
continue
else:
resume_on_repo = None
iteration_step_count += 1
continue
try:
repository = github_client.get_repo(repo.getKey())
repo.setRepoObject(repository)
except UnknownObjectException as e:
scream.log_warning('Repo with key + ' + key +
' not found, error({0}): {1}'.
format(e.status, e.data))
repos_reported_nonexist.write(key + os.linesep)
continue
except:
scream.log_warning('Repo with key + ' + key +
' not found, error({0}): {1}'.
format(e.status, e.data))
repos_reported_nonexist.write(key + os.linesep)
continue
iteration_step_count += 1
scream.ssay('Step no ' + str(iteration_step_count) +
'. Working on a repo: ' + key)
if resume_stage in [None, 'contributors']:
assert repo.getWatchersCount() is not None
# from this line move everything to a thread!
scream.say('Create instance of GeneralGetter with ID ' + str(thread_id_count) + ' and token ' + str(current_ghc_desc))
scream.log_debug('Make GeneralGetter object', True)
gg = GeneralGetter(thread_id_count, repository, repo, current_ghc)
scream.say('Creating instance of GeneralGetter complete')
scream.say('Appending thread to collection of threads')
threads.append(gg)
scream.say('Append complete, threads[] now have size: ' + str(len(threads)))
thread_id_count += 1
scream.log_debug('Starting thread ' + str(thread_id_count-1) + '....', True)
gg.start()
break
except UnknownObjectException as e:
scream.log_warning('Repo with key + ' + key +
' not found, error({0}): {1}'.
format(e.status, e.data), True)
repos_reported_nonexist.write(key + os.linesep)
continue
except GithubException as e:
scream.log_warning('Repo with key + ' + key +
' made exception in API, error({0}): {1}'.
format(e.status, e.data), True)
repos_reported_execution_error.write(key + os.linesep)
freeze(str(e) + ' in the main loop (most top try-catch)')
scream.say('Trying again with repo ' + str(key))
if show_trace:
scream.log_debug('Printing traceback stack', True)
traceback.print_stack()
scream.log_debug('Printing traceback exc pathway', True)
traceback.print_exc()
def freeze(message):
global sleepy_head_time
scream.log_warning('Sleeping for ' + str(sleepy_head_time) + ' seconds. Reason: ' + str(message), True)
time.sleep(sleepy_head_time)
def developer_revealed(thread_getter_instance, repository, repo, contributor):
global result_writer
developer_login = contributor.login
scream.log_debug(
'Assigning a contributor: ' + str(developer_login) + ' to a repo: ' +
str(repository.name), True)
developer_name = contributor.name
# 1 Ilosc osob, ktore dany deweloper followuje [FollowEvent]
developer_followers = contributor.followers
# 2 Ilosc osob, ktore followuja dewelopera [FollowEvent]
developer_following = contributor.following
developer_location = contributor.location
developer_total_private_repos = contributor.total_private_repos
developer_total_public_repos = contributor.public_repos
# 5. Ilosc repo, ktorych nie tworzyl, w ktorych jest team member [TeamAddEvent] [MemberEvent]
developer_collaborators = contributor.collaborators
# 6. Ilosc repo, ktorych nie tworzyl, w ktorych jest contributorem [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
developer_contributions = contributor.contributions
# - Ilosc projektow przez niego utworzonych
his_repositories = contributor.get_repos()
while True:
total_his_repositories = 0
total_his_stars = 0
total_his_watchers = 0
total_his_forks = 0
total_his_has_issues = 0
total_his_has_wiki = 0
total_his_open_issues = 0
total_network_count = 0
total_his_collaborators = 0
total_his_contributors = 0
if count___ == 'selenium':
total_his_commits = 0
total_his_branches = 0
total_his_releases = 0
total_his_issues = 0
total_his_pull_requests = 0
try:
for his_repo in his_repositories:
try:
total_his_repositories += 1
total_his_forks += his_repo.forks_count
total_his_stars += his_repo.stargazers_count
total_his_watchers += his_repo.watchers_count
total_his_has_issues += 1 if his_repo.has_issues else 0
total_his_has_wiki += 1 if his_repo.has_wiki else 0
total_his_open_issues += his_repo.open_issues
total_network_count += his_repo.network_count
if count___ == 'api':
# 3 Ilosc deweloperow, ktorzy sa w projektach przez niego utworzonych [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
total_his_contributors = None
while True:
try:
total_his_contributors = 0
#total_his_contributors = his_repo.get_contributors().totalCount -- this is buggy and will make errors
total_his_contributors += sum(
1 for temp_object in
his_repo.get_contributors())
break
except:
freeze(
'Exception in getting total_his_contributors'
)
if force_raise:
raise
assert total_his_contributors is not None
# 4 Ilosc kontrybutorow, ktorzy sa w projektach przez niego utworzonych
total_his_collaborators = None
while True:
try:
total_his_collaborators = 0
#total_his_collaborators = his_repo.get_collaborators().totalCount -- this is buggy and will make errors
total_his_collaborators += sum(
1 for temp_object in
his_repo.get_collaborators())
break
except:
freeze(
'Exception in getting total_his_collaborators'
)
if force_raise:
raise
assert total_his_collaborators is not None
elif count___ == 'selenium':
scream.say('Using selenium for thread about ' +
str(developer_login) + ' \'s repositories')
result = thread_getter_instance.analyze_with_selenium(
his_repo
) # wyciagnij statystyki przez selenium, i zwroc w tablicy:
# commits, branches, releases, contributors, issues, pull requests
if result['status'] == '404':
continue
if result['status'] == 'EMPTY':
continue
total_his_commits += result['commits']
total_his_branches += result['branches']
total_his_releases += result['releases']
total_his_issues += result['issues']
total_his_pull_requests += result['pulls']
total_his_contributors += result['contributors']
else: # hence it is only when not selenium is used
while True:
try:
his_contributors = set()
stats = his_repo.get_stats_contributors()
assert stats is not None
for stat in stats:
if str(stat.author.login).strip() in [
'None', ''
]:
continue
his_contributors.add(stat.author.login)
total_his_contributors += len(his_contributors)
break
except Exception as exc:
scream.log_warning(
'Not ready data while revealing details.. '
+ ', error({0})'.format(str(exc)), True)
freeze(
'StatsContribution not ready.. waiting for the server to provide good data'
)
if force_raise:
raise
except GithubException as e:
freeze(str(e) + ' in try per repo of x-dev repos')
if ("message"
in e.data) and (e.data["message"].strip()
== "Repository access blocked"):
scream.log_debug("It is a private repo.. Skip!")
continue
if force_raise:
raise
break
except Exception as e:
freeze(str(e) + ' in main loop of developer_revealed()')
his_repositories = contributor.get_repos()
if force_raise:
raise
# Developer company (if any given)
company = contributor.company
created_at = contributor.created_at
# Does the developer want to be hired?
hireable = contributor.hireable
scream.log_debug(
'Thread ' + str(thread_getter_instance.threadId) +
' Finished revealing contributor: ' + str(developer_login) +
' in a repo: ' + str(repository.name), True)
if show_trace:
scream.log_debug('Printing traceback stack', True)
traceback.print_stack()
scream.log_debug('Printing traceback exc pathway', True)
traceback.print_exc()
if not use_utf8:
result_writer.writerow([
str(repo.getUrl()),
str(repo.getName()),
str(repo.getOwner()),
str(repo.getStargazersCount()),
str(repo.getWatchersCount()),
str(developer_login),
(str(developer_name) if developer_name is not None else ''),
str(developer_followers),
str(developer_following),
str(developer_collaborators),
(str(company) if company is not None else ''),
str(developer_contributions),
str(created_at), (str(hireable) if hireable is not None else ''),
str(total_his_repositories),
str(total_his_stars),
str(total_his_collaborators),
str(total_his_contributors),
str(total_his_watchers),
str(total_his_forks),
str(total_his_has_issues),
str(total_his_has_wiki),
str(total_his_open_issues),
str(total_network_count),
(str(developer_location)
if developer_location is not None else ''),
str(developer_total_private_repos),
str(developer_total_public_repos),
str(total_his_issues),
str(total_his_pull_requests)
])
else:
result_writer.writerow([
repo.getUrl(),
repo.getName(),
repo.getOwner(),
str(repo.getStargazersCount()),
str(repo.getWatchersCount()), developer_login,
(developer_name if developer_name is not None else ''),
str(developer_followers),
str(developer_following),
str(developer_collaborators),
(company if company is not None else ''),
str(developer_contributions),
str(created_at), (str(hireable) if hireable is not None else ''),
str(total_his_repositories),
str(total_his_stars),
str(total_his_collaborators),
str(total_his_contributors),
str(total_his_watchers),
str(total_his_forks),
str(total_his_has_issues),
str(total_his_has_wiki),
str(total_his_open_issues),
str(total_network_count),
(developer_location if developer_location is not None else ''),
str(developer_total_private_repos),
str(developer_total_public_repos),
str(total_his_issues),
str(total_his_pull_requests)
])
scream.log_debug('Wrote row to CSV.', True)
def developer_revealed(thread_getter_instance, repository, repo, contributor):
global result_writer
global use_splinter
developer_login = contributor.login
scream.log_debug('Assigning a contributor: ' + str(developer_login) + ' to a repo: ' + str(repository.name), True)
developer_name = contributor.name
# 1 Ilosc osob, ktore dany deweloper followuje [FollowEvent]
developer_followers = contributor.followers
# 2 Ilosc osob, ktore followuja dewelopera [FollowEvent]
developer_following = contributor.following
developer_location = contributor.location
developer_total_private_repos = contributor.total_private_repos
developer_total_public_repos = contributor.public_repos
# 5. Ilosc repo, ktorych nie tworzyl, w ktorych jest team member [TeamAddEvent] [MemberEvent]
developer_collaborators = contributor.collaborators
# 6. Ilosc repo, ktorych nie tworzyl, w ktorych jest contributorem [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
developer_contributions = contributor.contributions
# - Ilosc projektow przez niego utworzonych
his_repositories = contributor.get_repos()
while True:
total_his_repositories = 0
total_his_stars = 0
total_his_watchers = 0
total_his_forks = 0
total_his_has_issues = 0
total_his_has_wiki = 0
total_his_open_issues = 0
total_network_count = 0
total_his_collaborators = 0
total_his_contributors = 0
if count___ == 'selenium':
total_his_commits = 0
total_his_branches = 0
total_his_releases = 0
total_his_issues = 0
total_his_pull_requests = 0
try:
for his_repo in his_repositories:
try:
total_his_repositories += 1
total_his_forks += his_repo.forks_count
total_his_stars += his_repo.stargazers_count
total_his_watchers += his_repo.watchers_count
total_his_has_issues += 1 if his_repo.has_issues else 0
total_his_has_wiki += 1 if his_repo.has_wiki else 0
total_his_open_issues += his_repo.open_issues
total_network_count += his_repo.network_count
if count___ == 'api':
# 3 Ilosc deweloperow, ktorzy sa w projektach przez niego utworzonych [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
total_his_contributors = None
while True:
try:
total_his_contributors = 0
#total_his_contributors = his_repo.get_contributors().totalCount -- this is buggy and will make errors
total_his_contributors += sum(1 for temp_object in his_repo.get_contributors())
break
except:
freeze('Exception in getting total_his_contributors')
if force_raise:
raise
assert total_his_contributors is not None
# 4 Ilosc kontrybutorow, ktorzy sa w projektach przez niego utworzonych
total_his_collaborators = None
while True:
try:
total_his_collaborators = 0
#total_his_collaborators = his_repo.get_collaborators().totalCount -- this is buggy and will make errors
total_his_collaborators += sum(1 for temp_object in his_repo.get_collaborators())
break
except:
freeze('Exception in getting total_his_collaborators')
if force_raise:
raise
assert total_his_collaborators is not None
elif count___ == 'selenium':
scream.say('Using selenium for thread about ' + str(developer_login) + ' \'s repositories')
if use_splinter:
result = thread_getter_instance.analyze_with_splinter(his_repo)
else:
result = thread_getter_instance.analyze_with_selenium(his_repo) # wyciagnij statystyki przez selenium, i zwroc w tablicy:
# commits, branches, releases, contributors, issues, pull requests
if result['status'] == '404':
continue
if result['status'] == 'EMPTY':
continue
total_his_commits += result['commits']
total_his_branches += result['branches']
total_his_releases += result['releases']
total_his_issues += result['issues']
total_his_pull_requests += result['pulls']
total_his_contributors += result['contributors']
else: # hence it is only when not selenium is used
while True:
try:
his_contributors = set()
stats = his_repo.get_stats_contributors()
assert stats is not None
for stat in stats:
if str(stat.author.login).strip() in ['None', '']:
continue
his_contributors.add(stat.author.login)
total_his_contributors += len(his_contributors)
break
except Exception as exc:
scream.log_warning('Not ready data while revealing details.. ' +
', error({0})'.format(str(exc)), True)
freeze('StatsContribution not ready.. waiting for the server to provide good data')
if force_raise:
raise
except GithubException as e:
freeze(str(e) + ' in try per repo of x-dev repos')
if ("message" in e.data) and (e.data["message"].strip() == "Repository access blocked"):
scream.log_debug("It is a private repo.. Skip!")
continue
if force_raise:
raise
break
except Exception as e:
freeze(str(e) + ' in main loop of developer_revealed()')
his_repositories = contributor.get_repos()
if force_raise:
raise
# Developer company (if any given)
company = contributor.company
created_at = contributor.created_at
# Does the developer want to be hired?
hireable = contributor.hireable
scream.log_debug('Thread ' + str(thread_getter_instance.threadId) +
' Finished revealing contributor: ' + str(developer_login) + ' in a repo: ' + str(repository.name), True)
if show_trace:
scream.log_debug('Printing traceback stack', True)
traceback.print_stack()
scream.log_debug('Printing traceback exc pathway', True)
traceback.print_exc()
if not use_utf8:
result_writer.writerow([str(repo.getUrl()), str(repo.getName()), str(repo.getOwner()),
str(repo.getStargazersCount()), str(repo.getWatchersCount()), str(developer_login),
(str(developer_name) if developer_name is not None else ''), str(developer_followers), str(developer_following),
str(developer_collaborators), (str(company) if company is not None else ''), str(developer_contributions),
str(created_at), (str(hireable) if hireable is not None else ''),
str(total_his_repositories), str(total_his_stars), str(total_his_collaborators), str(total_his_contributors),
str(total_his_watchers), str(total_his_forks), str(total_his_has_issues),
str(total_his_has_wiki), str(total_his_open_issues), str(total_network_count),
(str(developer_location) if developer_location is not None else ''),
str(developer_total_private_repos), str(developer_total_public_repos),
str(total_his_issues), str(total_his_pull_requests)])
else:
result_writer.writerow([repo.getUrl(), repo.getName(), repo.getOwner(), str(repo.getStargazersCount()), str(repo.getWatchersCount()), developer_login,
(developer_name if developer_name is not None else ''), str(developer_followers), str(developer_following),
str(developer_collaborators), (company if company is not None else ''), str(developer_contributions),
str(created_at), (str(hireable) if hireable is not None else ''),
str(total_his_repositories), str(total_his_stars), str(total_his_collaborators), str(total_his_contributors),
str(total_his_watchers), str(total_his_forks), str(total_his_has_issues),
str(total_his_has_wiki), str(total_his_open_issues), str(total_network_count),
(developer_location if developer_location is not None else ''),
str(developer_total_private_repos), str(developer_total_public_repos),
str(total_his_issues), str(total_his_pull_requests)])
scream.log_debug('Wrote row to CSV.', True)
def output_data(repo):
with open('repos.csv', 'ab') as output_csvfile:
scream.ssay('repos.csv opened for append..')
repowriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
rn = repo.getRepoObject()
rfc = repo.getForksCount()
rwc = repo.getWatchersCount()
rcc = repo.getContributorsCount()
rsc = repo.getSubscribersCount()
rstc = repo.getStargazersCount()
rlc = repo.getLabelsCount()
rcmc = repo.getCommitsCount()
rpc = repo.getPullsCount()
assert rfc.isdigit()
assert rwc.isdigit()
assert type(rcc) == int
assert type(rsc) == int
assert type(rstc) == int
assert type(rlc) == int
assert type(rcmc) == int
assert type(rpc) == int
tempv = (repo.getName(),
repo.getOwner(),
str(rfc), # this is always string representation of number
# ! str() allowed
str(rwc), # this is always string representation of number
# ! str() allowed
str(rcc), # this is always int ! str() allowed
str(rsc), # this is always int ! str() allowed
str(rstc), # this is always int ! str() allowed
str(rlc), # this is always int ! str() allowed
str(rcmc), # this is always int ! str() allowed
str(rpc), # this is always int ! str() allowed
rn.archive_url if rn.archive_url is not None else '',
rn.assignees_url if rn.assignees_url is not None else '',
rn.blobs_url if rn.blobs_url is not None else '',
rn.branches_url if rn.branches_url is not None else '',
rn.clone_url if rn.clone_url is not None else '',
rn.collaborators_url if rn.collaborators_url is not None else '',
rn.comments_url if rn.comments_url is not None else '',
rn.commits_url if rn.commits_url is not None else '',
rn.compare_url if rn.compare_url is not None else '',
rn.contents_url if rn.contents_url is not None else '',
rn.contributors_url if rn.contributors_url is not None else '',
str(rn.created_at) if rn.created_at is not None else '',
rn.default_branch if rn.default_branch is not None else '',
rn.description if rn.description is not None else '',
rn.events_url if rn.events_url is not None else '',
str(rn.fork) if rn.fork is not None else '',
rn.full_name if rn.full_name is not None else '',
rn.git_commits_url if rn.git_commits_url is not None else '',
rn.git_refs_url if rn.git_refs_url is not None else '',
rn.git_tags_url if rn.git_tags_url is not None else '',
str(rn.has_downloads) if rn.has_downloads is not None else '',
str(rn.has_wiki) if rn.has_wiki is not None else '',
rn.master_branch if rn.master_branch is not None else '')
repowriter.writerow(tempv)
with open('contributors.csv', 'ab') as output_csvfile:
scream.ssay('contributors.csv opened for append..')
contribwriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for contributor in repo.getContributors():
tempv = (repo.getName(),
repo.getOwner(),
contributor.login)
contribwriter.writerow(tempv)
with open('commits.csv', 'ab') as output_csvfile:
scream.ssay('commits.csv opened for append..')
commitswriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for commit in repo.getCommits():
tempv = (repo.getName(),
repo.getOwner(),
commit.sha,
(commit.author.login if commit.author is not None else ''),
(commit.committer.login if commit.committer is not None else ''),
commit.url,
commit.html_url,
commit.comments_url)
commitswriter.writerow(tempv)
if repo.getLanguages is not None:
with open('languages.csv', 'ab') as output_csvfile:
scream.ssay('languages.csv opened for append..')
langwriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for language in repo.getLanguages():
tempv = (repo.getName(),
repo.getOwner(),
language)
langwriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no languages[]')
if repo.getContributors() is not None:
with open('subscribers.csv', 'ab') as output_csvfile:
scream.ssay('subscribers.csv opened for append..')
subscriberswriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for subscriber in repo.getContributors():
tempv = (repo.getName(),
repo.getOwner(),
(subscriber.login if subscriber.login is not None else ''),
(subscriber.bio if subscriber.bio is not None else ''),
(subscriber.blog if subscriber.blog is not None else ''),
str(subscriber.collaborators),
(subscriber.company if subscriber.company is not None else ''),
str(subscriber.contributions),
str(subscriber.followers),
str(subscriber.following))
subscriberswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no subscribers[]')
if repo.getLabels() is not None:
with open('labels.csv', 'ab') as output_csvfile:
scream.ssay('labels.csv opened for append..')
labelswriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for label in repo.getLabels():
tempv = (repo.getName(),
repo.getOwner(),
label.name,
label.color)
labelswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no labels[]')
if repo.getIssues() is not None:
with open('issues.csv', 'ab') as output_csvfile:
scream.ssay('issues.csv opened for append..')
issueswriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for issue in repo.getIssues():
assert (type(issue.id) == int or issue.id is None)
assert (type(issue.number) == int or issue.number is None)
tempv = (repo.getName(),
repo.getOwner(),
(issue.assignee.login if issue.assignee is not None else ''),
(' '.join(issue.body.splitlines()) if issue.body is not None else ''),
(issue.closed_at if issue.closed_at is not None else ''),
(issue.closed_by.login if issue.closed_by is not None else ''),
str(issue.id),
str(issue.number),
(issue.title if issue.title is not None else ''))
issueswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no issues[]')
if repo.getPulls() is not None:
with open('pulls.csv', 'ab') as output_csvfile:
scream.ssay('pulls.csv opened for append..')
pullswriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(output_csvfile, dialect=MyDialect)
for pull in repo.getPulls():
tempv = (repo.getName(),
repo.getOwner(),
str(pull.additions), # is always int
(pull.assignee.login if pull.assignee is not None else ''),
(' '.join(pull.body.splitlines()) if pull.body is not None else ''),
str(pull.changed_files), # is always int
(str(pull.closed_at) if pull.closed_at is not None else ''),
str(pull.comments), # is always int
(pull.comments_url if pull.comments_url is not None else ''),
(str(pull.created_at) if pull.created_at is not None else ''),
str(pull.deletions), # is always int
(pull.diff_url if pull.diff_url is not None else ''),
(pull.html_url if pull.html_url is not None else ''),
str(pull.id), # is always int
(pull.issue_url if pull.issue_url is not None else ''),
(pull.merge_commit_sha if pull.merge_commit_sha is not None else ''),
str(pull.mergeable), # is always boolean
(pull.mergeable_state if pull.mergeable_state is not None else ''),
str(pull.merged), # is always boolean
(str(pull.merged_at) if pull.merged_at is not None else ''),
str(pull.number),
(pull.patch_url if pull.patch_url is not None else ''),
(pull.review_comment_url if pull.review_comment_url is not None else ''),
str(pull.review_comments), # is always int
(pull.review_comments_url if pull.review_comments_url is not None else ''),
(pull.state if pull.state is not None else ''),
(' '.join(pull.title.splitlines()) if pull.title is not None else ''),
(str(pull.updated_at) if pull.updated_at is not None else ''),
(pull.user.login if pull.user is not None else ''))
pullswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no pulls[]')
for filename in file_names:
scream.say('------ WORKING WITH FILE : ' + filename)
filename_ = 'data/' if sys.platform == 'linux2' else 'data\\'
filename__ = filename_ + filename + '.csv'
with open(filename__, 'rb') as source_csvfile:
reposReader = csv.reader(source_csvfile,
delimiter=',')
reposReader.next()
for row in reposReader:
scream.log('Processing row: ' + str(row))
name = row[0]
owner = row[1]
#here eleminate repos without owner, rly
if len(owner.strip()) < 1:
scream.log_warning('Skipping orphan repo: ' + name)
continue
#print 'length < 1'
'12. Liczba Fork'
forks = row[2]
watchers = row[3]
key = owner + '/' + name
scream.log('Key built: ' + key)
repo = MyRepository()
repo.setKey(key)
repo.setInitials(name, owner, watchers, forks)
#check here if repo dont exist already in dictionary!
if key in repos:
if not ((resume_on_repo_name == repo.getName()) and
(resume_on_repo_owner == repo.getOwner())):
iteration_step_count += 1
continue
else:
resume_on_repo = None
iteration_step_count += 1
continue
try:
repository = github_client.get_repo(repo.getKey())
repo.setRepoObject(repository)
except UnknownObjectException as e:
scream.log_warning(
'Repo with key + ' + key +
' not found, error({0}): {1}'.format(e.status, e.data))
repos_reported_nonexist.write(key + os.linesep)
continue
except:
scream.log_warning(
'Repo with key + ' + key +
' not found, error({0}): {1}'.format(e.status, e.data))
repos_reported_nonexist.write(key + os.linesep)
continue
iteration_step_count += 1
scream.ssay('Step no ' + str(iteration_step_count) +
'. Working on a repo: ' + key)
if resume_stage in [None, 'contributors']:
"ctl00_ContentPlaceHolder1_" +
"LabelFailedSearchedFor"})
if failed is not None:
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
gender_tag = local_soup.find("span",
{"id":
"ctl00_ContentPlaceHolder1_" +
"LabelGenderFound"})
if ((gender_tag is not None) and (gender_tag.contents is not None) and (len(gender_tag.contents) > 0)):
gender = gender_tag.contents[0].string
scream.say(gender)
persist_users[s] = s + ',' + fullname.strip() + ',' + gender
return s + ',' + fullname.strip() + ',' + gender
else:
scream.log_warning('Something really wrong, on result page there ' +
'was no not-found label neither a proper result')
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
else:
persist_users[s] = s + ',' + fullname.strip()
return s + ',' + fullname.strip()
else:
scream.say('Fullname not provided')
persist_users[s] = None
return s
# This method tries to get HTML discussion site again again
# when there is a problem with finding by th BS
# proper html tags which allways indicate author
# or discussion content - a must content to exist
def freeze(message):
global sleepy_head_time
scream.log_warning('Sleeping for ' + str(sleepy_head_time) + ' seconds. Reason: ' + str(message), True)
time.sleep(sleepy_head_time)
for filename in file_names:
scream.say('------ WORKING WITH FILE : ' + filename)
filename_ = 'data/' if sys.platform == 'linux2' else 'data\\'
filename__ = filename_ + filename + '.csv'
with open(filename__, 'rb') as source_csvfile:
reposReader = csv.reader(source_csvfile, delimiter=',')
reposReader.next()
for row in reposReader:
scream.log('Processing row: ' + str(row))
name = row[0]
owner = row[1]
#here eleminate repos without owner, rly
if len(owner.strip()) < 1:
scream.log_warning('Skipping orphan repo: ' + name)
continue
#print 'length < 1'
'12. Liczba Fork'
forks = row[2]
watchers = row[3]
key = owner + '/' + name
scream.log('Key built: ' + key)
repo = MyRepository()
repo.setKey(key)
repo.setInitials(name, owner, watchers, forks)
#check here if repo dont exist already in dictionary!
if key in repos:
gg = GeneralGetter(thread_id_count, repository, repo,
current_ghc)
scream.say('Creating instance of GeneralGetter complete')
scream.say('Appending thread to collection of threads')
threads.append(gg)
scream.say('Append complete, threads[] now have size: ' +
str(len(threads)))
thread_id_count += 1
scream.log_debug(
'Starting thread ' + str(thread_id_count - 1) + '....',
True)
gg.start()
break
except UnknownObjectException as e:
scream.log_warning(
'Repo with key + ' + key +
' not found, error({0}): {1}'.format(e.status, e.data),
True)
repos_reported_nonexist.write(key + os.linesep)
continue
except GithubException as e:
scream.log_warning(
'Repo with key + ' + key +
' made exception in API, error({0}): {1}'.format(
e.status, e.data), True)
if ("message" in e.data) and (e.data["message"].strip()
== "Repository access blocked"):
scream.log_debug("It is now a private repo.. Skip!", True)
continue
repos_reported_execution_error.write(key + os.linesep)
freeze(str(e) + ' in the main loop (most top try-catch)')
scream.say('Trying again with repo ' + str(key))
def output_data(repo):
with open('repos.csv', 'ab') as output_csvfile:
scream.ssay('repos.csv opened for append..')
repowriter = UnicodeWriter(output_csvfile) if use_utf8 else csv.writer(
output_csvfile, dialect=MyDialect)
rn = repo.getRepoObject()
rfc = repo.getForksCount()
rwc = repo.getWatchersCount()
rcc = repo.getContributorsCount()
rsc = repo.getSubscribersCount()
rstc = repo.getStargazersCount()
rlc = repo.getLabelsCount()
rcmc = repo.getCommitsCount()
rpc = repo.getPullsCount()
assert rfc.isdigit()
assert rwc.isdigit()
assert type(rcc) == int
assert type(rsc) == int
assert type(rstc) == int
assert type(rlc) == int
assert type(rcmc) == int
assert type(rpc) == int
tempv = (
repo.getName(),
repo.getOwner(),
str(rfc), # this is always string representation of number
# ! str() allowed
str(rwc), # this is always string representation of number
# ! str() allowed
str(rcc), # this is always int ! str() allowed
str(rsc), # this is always int ! str() allowed
str(rstc), # this is always int ! str() allowed
str(rlc), # this is always int ! str() allowed
str(rcmc), # this is always int ! str() allowed
str(rpc), # this is always int ! str() allowed
rn.archive_url if rn.archive_url is not None else '',
rn.assignees_url if rn.assignees_url is not None else '',
rn.blobs_url if rn.blobs_url is not None else '',
rn.branches_url if rn.branches_url is not None else '',
rn.clone_url if rn.clone_url is not None else '',
rn.collaborators_url if rn.collaborators_url is not None else '',
rn.comments_url if rn.comments_url is not None else '',
rn.commits_url if rn.commits_url is not None else '',
rn.compare_url if rn.compare_url is not None else '',
rn.contents_url if rn.contents_url is not None else '',
rn.contributors_url if rn.contributors_url is not None else '',
str(rn.created_at) if rn.created_at is not None else '',
rn.default_branch if rn.default_branch is not None else '',
rn.description if rn.description is not None else '',
rn.events_url if rn.events_url is not None else '',
str(rn.fork) if rn.fork is not None else '',
rn.full_name if rn.full_name is not None else '',
rn.git_commits_url if rn.git_commits_url is not None else '',
rn.git_refs_url if rn.git_refs_url is not None else '',
rn.git_tags_url if rn.git_tags_url is not None else '',
str(rn.has_downloads) if rn.has_downloads is not None else '',
str(rn.has_wiki) if rn.has_wiki is not None else '',
rn.master_branch if rn.master_branch is not None else '')
repowriter.writerow(tempv)
with open('contributors.csv', 'ab') as output_csvfile:
scream.ssay('contributors.csv opened for append..')
contribwriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for contributor in repo.getContributors():
tempv = (repo.getName(), repo.getOwner(), contributor.login)
contribwriter.writerow(tempv)
with open('commits.csv', 'ab') as output_csvfile:
scream.ssay('commits.csv opened for append..')
commitswriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for commit in repo.getCommits():
tempv = (repo.getName(), repo.getOwner(), commit.sha,
(commit.author.login if commit.author is not None else
''), (commit.committer.login
if commit.committer is not None else ''),
commit.url, commit.html_url, commit.comments_url)
commitswriter.writerow(tempv)
if repo.getLanguages is not None:
with open('languages.csv', 'ab') as output_csvfile:
scream.ssay('languages.csv opened for append..')
langwriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for language in repo.getLanguages():
tempv = (repo.getName(), repo.getOwner(), language)
langwriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no languages[]')
if repo.getContributors() is not None:
with open('subscribers.csv', 'ab') as output_csvfile:
scream.ssay('subscribers.csv opened for append..')
subscriberswriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for subscriber in repo.getContributors():
tempv = (
repo.getName(), repo.getOwner(),
(subscriber.login if subscriber.login is not None else ''),
(subscriber.bio if subscriber.bio is not None else ''),
(subscriber.blog if subscriber.blog is not None else ''),
str(subscriber.collaborators),
(subscriber.company if subscriber.company is not None else
''), str(subscriber.contributions),
str(subscriber.followers), str(subscriber.following))
subscriberswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no subscribers[]')
if repo.getLabels() is not None:
with open('labels.csv', 'ab') as output_csvfile:
scream.ssay('labels.csv opened for append..')
labelswriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for label in repo.getLabels():
tempv = (repo.getName(), repo.getOwner(), label.name,
label.color)
labelswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no labels[]')
if repo.getIssues() is not None:
with open('issues.csv', 'ab') as output_csvfile:
scream.ssay('issues.csv opened for append..')
issueswriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for issue in repo.getIssues():
assert (type(issue.id) == int or issue.id is None)
assert (type(issue.number) == int or issue.number is None)
tempv = (repo.getName(), repo.getOwner(),
(issue.assignee.login if issue.assignee is not None
else ''), (' '.join(issue.body.splitlines())
if issue.body is not None else ''),
(issue.closed_at if issue.closed_at is not None else
''), (issue.closed_by.login
if issue.closed_by is not None else ''),
str(issue.id), str(issue.number),
(issue.title if issue.title is not None else ''))
issueswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no issues[]')
if repo.getPulls() is not None:
with open('pulls.csv', 'ab') as output_csvfile:
scream.ssay('pulls.csv opened for append..')
pullswriter = UnicodeWriter(
output_csvfile) if use_utf8 else csv.writer(output_csvfile,
dialect=MyDialect)
for pull in repo.getPulls():
tempv = (
repo.getName(),
repo.getOwner(),
str(pull.additions), # is always int
(pull.assignee.login if pull.assignee is not None else ''),
(' '.join(pull.body.splitlines())
if pull.body is not None else ''),
str(pull.changed_files), # is always int
(str(pull.closed_at)
if pull.closed_at is not None else ''),
str(pull.comments), # is always int
(pull.comments_url
if pull.comments_url is not None else ''),
(str(pull.created_at)
if pull.created_at is not None else ''),
str(pull.deletions), # is always int
(pull.diff_url if pull.diff_url is not None else ''),
(pull.html_url if pull.html_url is not None else ''),
str(pull.id), # is always int
(pull.issue_url if pull.issue_url is not None else ''),
(pull.merge_commit_sha
if pull.merge_commit_sha is not None else ''),
str(pull.mergeable), # is always boolean
(pull.mergeable_state
if pull.mergeable_state is not None else ''),
str(pull.merged), # is always boolean
(str(pull.merged_at)
if pull.merged_at is not None else ''),
str(pull.number),
(pull.patch_url if pull.patch_url is not None else ''),
(pull.review_comment_url
if pull.review_comment_url is not None else ''),
str(pull.review_comments), # is always int
(pull.review_comments_url
if pull.review_comments_url is not None else ''),
(pull.state if pull.state is not None else ''),
(' '.join(pull.title.splitlines())
if pull.title is not None else ''),
(str(pull.updated_at)
if pull.updated_at is not None else ''),
(pull.user.login if pull.user is not None else ''))
pullswriter.writerow(tempv)
else:
scream.log_warning('Repo ' + repo.getName() + ' has no pulls[]')
resume_on_repo_owner = resume_on_repo.split(',')[1]
if not ((resume_on_repo_name == repo.getName()) and
(resume_on_repo_owner == repo.getOwner())):
iteration_step_count += 1
continue
try:
repository_object = repos[key]
repository = gh.get_repo(key)
print 'getting contributors'
contributors = repository.get_contributors()
repo.setRepoObject(repository)
except UnknownObjectException as e:
scream.log_warning('Repo with key + ' + key +
' not found, error({0}): {1}'.
format(e.status, e.data))
repos_reported_nonexist.append(key)
continue
try:
languages = repository.get_languages()
print str(languages)
print repository.language
print repository.languages_url
except GithubException as gite:
scream.log_warning('GithubException while gettings langs in + ' + key +
' , error({0}): {1}'.
format(e.status, e.data))
continue
def developer_revealed(thread_getter_instance, repository, repo, contributor):
global result_writer
global result_punch_card_writer
assert result_punch_card_writer is not None
developer_login = contributor.login
scream.log_debug('Assigning a contributor: ' + str(developer_login) + ' to a repo: ' + str(repository.name), True)
developer_name = contributor.name
# 1. Ilosc osob, ktore dany deweloper followuje [FollowEvent]
developer_followers = contributor.followers
# 2. Ilosc osob, ktore followuja dewelopera [FollowEvent]
developer_following = contributor.following
developer_location = contributor.location
developer_total_private_repos = contributor.total_private_repos
developer_total_public_repos = contributor.public_repos
# 5a. Ilosc repo, w ktorych jest team member [TeamAddEvent] [MemberEvent]
developer_collaborators = contributor.collaborators
scream.say('Developer collaborators count is: ' + str(developer_collaborators))
# 6a. Ilosc repo, w ktorych jest contributorem [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
developer_contributions = contributor.contributions
scream.say('Developer contributions count is: ' + str(developer_contributions))
# his_repositories - Ilosc projektow przez niego utworzonych / ktorych jest wlascicielem
his_repositories = contributor.get_repos()
# 17. Czy commituje w godzinach pracy (zaleznie od strefy czasowej)?
scream.log_debug("Starting to analyze OSRC card for user: "******"Histogram for hours for user: "******"pod repozytorium"
while True:
try:
trying_to_get_stats += 1
stats = his_repo.get_stats_contributors()
status_code__ = get_status_code('https://api.github.com/repos/' + his_repo.full_name + '/stats/contributors')
if status_code__ != 204:
for s in stats:
ad___c = 0
ad___a = 0
ad___d = 0
for w in s.weeks:
ad___c += w.c
ad___a += w.a
ad___d += w.d
if s.author.login not in his_contributors:
his_contributors.add(s.author.login)
result_punch_card_writer.writerow([str(his_repo.owner.login), str(his_repo.name),
str(developer_login), str(s.author.login), str(s.total), str(ad___c), str(ad___a), str(ad___d)])
else:
scream.log_debug('The subrepo is empty, thus no statistics (punchcard) generated this time', True)
break
except GithubException as e:
freeze(str(e) + ' his_repo.get_stats_contributors(). Unexpected error with getting stats.')
if ("message" in e.data) and (e.data["message"].strip() == "Repository access blocked"):
scream.log_debug("It is a private repo.. Skip!", True)
break
if force_raise:
raise
except TypeError as e:
scream.log_warning('This was stats attempt no: ' + str(trying_to_get_stats), True)
freeze(str(e) + ' his_repo.get_stats_contributors(). Punch-card not ready?')
# probably punch card not ready
if force_raise:
raise
except Exception as e:
freeze(str(e) + ' his_repo.get_stats_contributors(). Punch-card not ready?')
# probably punch card not ready
if force_raise:
raise
# 6. Ilosc repo, ktorych nie tworzyl, w ktorych jest deweloperem
if developer_login in his_contributors:
self_contributing += 1
# 5. Ilosc repo, ktorych nie tworzyl, w ktorych jest team member
subrepo_collaborators = his_repo.get_collaborators()
for collaborator in subrepo_collaborators:
total_his_collaborators += 1
if developer_login == collaborator.login:
self_collaborating += 1
# All elements paginated through his_repositories, thus we can safely break loop
break
except GithubException as e:
freeze('While getting subrepo details, ' + str(e) + ' in element his_repo in his_repositories')
if ("message" in e.data) and (e.data["message"].strip() == "Repository access blocked"):
scream.log_debug("It is a private repo.. Skip!")
continue
if force_raise:
raise
except TypeError as e:
freeze('While getting subrepo details, ' + str(e) + ' in element his_repo in his_repositories. Quota depleted?')
# probably punch card not ready
if force_raise:
raise
except Exception as e:
freeze('While getting subrepo details, ' + str(e) + ' in element his_repo in his_repositories. Quota depleted?')
# probably punch card not ready
if force_raise:
raise
total_his_contributors = len(his_contributors)
# 5. Ilosc repo, ktorych nie tworzyl, w ktorych jest team member [TeamAddEvent] [MemberEvent]
# tutaj od wartosci developer_collaborators wystarczy odjac wystapienia loginu w podrepo.get_collaborators()
developer_foreign_collaborators = (developer_collaborators if developer_collaborators is not None else 0) - self_collaborating
# 6. Ilosc repo, ktorych nie tworzyl, w ktorych jest contributorem [PushEvent] [IssuesEvent] [PullRequestEvent] [GollumEvent]
# tutaj od wartosci developer_contributions wystarczy odjac wystapienia loginu w podrepo.get_contributions()
developer_foreign_contributions = developer_contributions - self_contributing
# All properties checked for a dev, thus we can safely break loop
break
except Exception as e:
freeze('Error ' + str(e) + ' in for his_repo in his_repositories loop. Will start the subrepo analysis from the beginning.')
his_repositories = contributor.get_repos()
if force_raise:
raise
# Developer company (if any given)
company = contributor.company
created_at = contributor.created_at
# Does the developer want to be hired?
hireable = contributor.hireable
disk_usage = contributor.disk_usage
public_gists = contributor.public_gists
owned_private_repos = contributor.owned_private_repos
total_private_repos = contributor.total_private_repos
scream.log_debug('Thread ' + str(thread_getter_instance) +
' Finished revealing contributor: ' + str(developer_login) + ' in a repo: ' + str(repository.name), True)
if show_trace:
scream.log_debug('Printing traceback stack', True)
traceback.print_stack()
scream.log_debug('Printing traceback exc pathway', True)
traceback.print_exc()
if not use_utf8:
result_writer.writerow([str(repo.getUrl()), str(repo.getName()), str(repo.getOwner()),
str(repo.getStargazersCount()), str(repo.getWatchersCount()),
str(repo.getCreatedAt()), str(repo.getDefaultBranch()), str(repo.getDescription()),
str(repo.getIsFork()), str(repo.getForks()), str(repo.getForksCount()),
str(repo.getHasDownloads()), str(repo.getHasWiki()), str(repo.getHasIssues()),
str(repo.getLanguage()), str(repo.getMasterBranch()), str(repo.getNetworkCount()), str(repo.getOpenedIssues()),
str(repo.getOrganization()), str(repo.getPushedAt()), str(repo.getUpdatedAt()), str(repo.getPullsCount()),
str(total_his_contributors), str(total_his_collaborators), str(developer_foreign_collaborators),
str(developer_foreign_contributions), str(total_his_issues), str(total_his_pull_requests),
str(developer_login),
str(developer_name if developer_name is not None else ''), str(developer_followers), str(developer_following),
str(developer_collaborators), str(company if company is not None else ''), str(developer_contributions),
str(created_at), str(hireable if hireable is not None else ''),
str(developer_location if developer_location is not None else ''),
str(developer_total_private_repos), str(developer_total_public_repos),
str(developer_works_during_bd), str(developer_works_period), str(disk_usage),
str(public_gists), str(owned_private_repos), str(total_private_repos)])
else:
result_writer.writerow([repo.getUrl(), repo.getName(), repo.getOwner(), str(repo.getStargazersCount()), str(repo.getWatchersCount()),
str(repo.getCreatedAt()), repo.getDefaultBranch(), repo.getDescription() if repo.getDescription() is not None else '',
str(repo.getIsFork()), str(repo.getForks()), str(repo.getForksCount()),
str(repo.getHasDownloads()), str(repo.getHasWiki()), str(repo.getHasIssues()),
repo.getLanguage() if repo.getLanguage() is not None else '',
repo.getMasterBranch() if repo.getMasterBranch() is not None else '',
str(repo.getNetworkCount()), str(repo.getOpenedIssues()),
repo.getOrganization() if repo.getOrganization() is not None else '',
str(repo.getPushedAt()), str(repo.getUpdatedAt()), str(repo.getPullsCount()),
str(total_his_contributors), str(total_his_collaborators), str(developer_foreign_collaborators),
str(developer_foreign_contributions), str(total_his_issues), str(total_his_pull_requests),
developer_login,
developer_name if developer_name is not None else '', str(developer_followers), str(developer_following),
str(developer_collaborators), company if company is not None else '', str(developer_contributions),
str(created_at), str(hireable) if hireable is not None else '',
developer_location if developer_location is not None else '',
str(developer_total_private_repos), str(developer_total_public_repos),
str(developer_works_during_bd), str(developer_works_period), str(disk_usage),
str(public_gists), str(owned_private_repos), str(total_private_repos)])
scream.log_debug('Wrote row to CSV.', True)
