-
Notifications
You must be signed in to change notification settings - Fork 0
/
reportificate.py
executable file
·233 lines (181 loc) · 5.75 KB
/
reportificate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
#!/usr/bin/python
#
# github API abusing script for report metrics
#
# C. Orchard - 29/11/2012
#
import itertools
import getpass
import os
import datetime
import httplib
from operator import itemgetter
from contextlib import closing
from fnmatch import fnmatchcase
from subprocess import Popen, PIPE
from urlparse import urljoin
from collections import defaultdict, deque
import percache
import keyring
import dateutil.parser
import requests
import matplotlib.pyplot as plt
import numpy as np
from xdg import BaseDirectory
from six.moves import configparser
github_url = "https://api.github.com/"
repo_url = "repos/chrissorchard/malucrawl/commits"
commit_url = "repos/chrissorchard/malucrawl/commits/"
status_url = "rate_limit"
full_repo_url = urljoin(github_url, repo_url)
path = "doc/report"
valid_files = "doc/report/*.tex"
working_dir = os.path.dirname(os.path.realpath(__file__))
def get_auth():
def check_auth(auth):
return requests.head(full_repo_url, auth=auth).status_code != httplib.UNAUTHORIZED
# config file init
config_file = os.path.join(BaseDirectory.save_config_path("malucrawl_reportificate"), "settings.ini")
config = configparser.SafeConfigParser()
config.read(config_file)
if not config.has_section('auth'):
config.add_section('auth')
try:
username = config.get('auth', 'username')
except configparser.NoOptionError:
username = raw_input("Username: ")
password = keyring.get_password(github_url, username)
if password is None:
password = getpass.getpass()
while not check_auth((username, password)):
print "Authorization Failed"
username = raw_input("Username: ")
password = getpass.getpass()
keyring.set_password(github_url, username, password)
config.set('auth', 'username', username)
config.write(open(config_file, 'w'))
return (username, password)
session = requests.Session()
session.auth = get_auth()
def commits_generator():
url = full_repo_url
param = {
'path': path,
'per_page': '100'
}
while url:
r = session.get(
url,
params=param
)
yield r.json()
url = r.links.get("next", {"url": False})["url"]
param = {}
with closing(percache.Cache(
os.path.join(BaseDirectory.save_cache_path("malucrawl_reportificate"), "cache")
)) as cache:
@cache
def get_commit_details(commit_url):
return session.get(commit_url).json()
@cache
def count_words_in_tree(tree_url):
return sum(
map(
lambda tree: blob_lacount(tree["url"]),
itertools.ifilter(
lambda tree: tree["type"] == "blob" and fnmatchcase(tree["path"], valid_files),
session.get(tree_url, params={"recursive": 1}).json()["tree"]
)
)
)
@cache
def blob_lacount(blob_url):
response = session.get(blob_url,
headers={"Accept": "application/vnd.github.v3.raw"}
)
cmd = os.path.join(working_dir, "texcount.pl")
count_proc = Popen([cmd, "-1", "-sum", "-"], stdin=PIPE, stdout=PIPE)
try:
stdout, stderr = count_proc.communicate(input=response.content)
return int(stdout.splitlines(False)[0])
except (ValueError):
return 0
all_commits = itertools.chain.from_iterable(commits_generator())
# Count the total number of words in each commit
# Append to data: the author of the commit, the time it was made
# and the absolute word count at that commit.
#
# Note: This no-longer counts deltas
data = []
for commit_number, commit in enumerate(all_commits):
print (commit_number, commit["sha"])
if commit["committer"] is None:
commit["author"] = {
"login": u"nafisehvahabi"
}
data.append(
(
commit["author"]["login"],
dateutil.parser.parse(commit["commit"]["committer"]["date"]),
count_words_in_tree(commit["commit"]["tree"]["url"])
)
)
print session.get(urljoin(github_url, status_url)).json()
#
# Graphing
#
sdata = sorted(data, key=itemgetter(1))
mindate = sdata[0][1].date()
maxdate = sdata[-1][1].date()
delta = maxdate - mindate
date_list = [mindate + datetime.timedelta(days=x)
for x in range(0, delta.days + 1)]
#make four lists for y values
# associate deltas with the relevant username
# sum deltas with the same username and date
namecount = defaultdict(lambda: defaultdict(lambda: 0))
nametotal = defaultdict(lambda: 0)
old_words = 0
for name, dt, count in sdata:
delta = count - old_words
old_words = count
namecount[name][dt.date()] += delta
nametotal[name] += delta
nameadd = {}
colors = deque(["b", "g", "r", "c", "m", "y"])
labels = []
bars = []
barsum = np.zeros(len(date_list))
sorted_name_count = sorted(
namecount.items(),
key=lambda item: nametotal[item[0]],
reverse=True
)
for name, course_date_count in sorted_name_count:
amounts = np.zeros(len(date_list))
running_total = np.zeros(len(date_list))
for i, date in enumerate(date_list):
amounts[i] = course_date_count.get(date, 0)
total = 0
for i, count in enumerate(amounts):
total += count
running_total[i] = total
nameadd[name] = running_total
labels.append(name)
bars.append(plt.bar(
date_list,
nameadd[name],
bottom=barsum,
color=colors[0]
)[0]
)
colors.rotate(-1)
barsum += running_total
print date_list
print nameadd
plt.xticks(rotation='vertical')
plt.ylabel('Words')
plt.title('Report Word Count Breakdown')
plt.legend(bars, labels, loc="upper left")
plt.subplots_adjust(bottom=.2)
plt.show()