def plot_prs_merged(created_ats, prs, repository):
"""Plot stats on merged PRs."""
init_year = min(created_ats).year
curr_year = datetime_parser('TODAY').year
start_end_years = {}
for year in range(init_year, curr_year + 1):
start_year = datetime_parser('%s-01-01' % year)
end_year = datetime_parser('%s-12-31 23:59:59' % year)
start_end_years[year] = (start_year, end_year)
print((year, len([x for x in created_ats if x >= start_year and x <= end_year])))
prs_by_year = {}
for (created_at, pr) in zip(created_ats, prs):
for year, (start_year, end_year) in start_end_years.items():
if created_at >= start_year and created_at <= end_year:
prs_by_year.setdefault(year, []).append(pr)
gh_logins = ['boegel', 'verdurin', 'pescobar', 'vanzod', 'wpoely86', 'JensTimmerman', 'migueldiascosta']
maintainers = ['boegel', 'verdurin', 'pescobar', 'vanzod', 'wpoely86', 'migueldiascosta', 'akesandgren',
'BartOldeman', 'damianam', 'ocaisa', 'Micket', 'zao', 'smoors', 'lexming',
'casparvl', 'branfosj']
# (+ wpoely86 in 2016...)
hpcugent = ['JensTimmerman', 'Caylo', 'stdweird', 'itkovian', 'piojo', 'hpcugent', 'nudded', 'boegel']
gh_logins_bis = gh_logins + ['hajgato', 'fgeorgatos', 'RvDijk', 'JackPerdue', 'smoors', 'geimer', 'SimonPinches']
gh_logins_bis += ['Helios07', 'cstackpole', 'akesandgren', 'rubendibattista', 'BartOldeman', 'damianam', 'ocaisa',
'stdweird', 'nudded', 'piojo', 'Caylo', 'hpcugent', 'Darkless012', 'zarybnicky', 'deniskristak']
for year in sorted(prs_by_year.keys()):
prs_cnt = len(prs_by_year[year])
prs_cnt_maintainers = len([pr for pr in prs_by_year[year] if pr['user']['login'] in maintainers])
prs_cnt_hpcugent = len([pr for pr in prs_by_year[year] if pr['user']['login'] in hpcugent])
prs_cnt_by = {}
for gh_login in gh_logins_bis:
prs_cnt_by[gh_login] = len([pr for pr in prs_by_year[year] if pr['user']['login'] == gh_login])
merged_prs = [pr for pr in prs_by_year[year] if pr.get('is_merged', False)]
merged_cnt = len(merged_prs)
merged_cnt_by = {}
for gh_login in gh_logins:
merged_cnt_by[gh_login] = len([pr for pr in merged_prs if pr['merged_by']['login'] == gh_login])
closed_cnt = len([pr for pr in prs_by_year[year] if pr['state'] == 'closed'])
open_cnt = len([pr for pr in prs_by_year[year] if pr['state'] == 'open'])
print('* %s: %s PRs' % (year, prs_cnt))
print('* %s unique contributors' % len(nub(pr['user']['login'] for pr in prs_by_year[year])))
print('* PRs by maintainers: %s' % prs_cnt_maintainers)
print('* PRs by HPC-UGent: %s' % prs_cnt_hpcugent)
for gh_login in gh_logins_bis:
print('- PRs by %s: ' % gh_login, prs_cnt_by[gh_login])
print('- merged: ', merged_cnt)
for gh_login in gh_logins:
print('- merged by %s: ' % gh_login, merged_cnt_by[gh_login])
print('- closed: ', closed_cnt - merged_cnt)
print('- open: ', open_cnt)
new_pr_tag = "created using `eb --new-pr`"
print('- opened with --new-pr: ', len([pr for pr in prs_by_year[year] if new_pr_tag in (pr['body'] or '')]))
print('')
def plot_pr_stats(prs, go):
""" Create plot overview of prs, save in pdf format"""
created_ats = [datetime_parser(pr['created_at'].split('T')[0]) for pr in prs]
closed_ats = [datetime_parser((pr['closed_at'] or 'T').split('T')[0] or 'ENDNEXTMONTH') for pr in prs]
authors = [pr['user']['login'] for pr in prs]
print('Plotting...')
plot_historic_PR_ages(created_ats, closed_ats, go.options.repository)
plot_open_closed_PRs(created_ats, closed_ats, go.options.repository)
plot_prs_by_author(created_ats, authors, go.options.repository)
print_prs_uniq_authors(created_ats, authors, go.options.repository)
plot_prs_merged(created_ats, prs, go.options.repository)
def test_datetime_parser(self):
"""Test the date_parser"""
testdate = datetime.datetime(1970, 1, 1)
self.assertEqual(datetime_parser('1970-01-01') , testdate)
self.assertEqual(datetime_parser('1970-1-1'), testdate)
today = datetime.datetime.today()
beginthismonth = datetime.datetime(today.year, today.month, 1, 12, 1)
self.assertEqual(datetime_parser('BEGINTHISMONTH 12:1') , beginthismonth)
endapril = datetime.datetime(today.year, 4, 30, 12, 1, 1, 1)
self.assertEqual(datetime_parser('ENDAPRIL 12:01:01.000001') , endapril)
def test_datetime_parser(self):
"""Test the date_parser"""
testdate = datetime.datetime(1970, 1, 1)
self.assertEqual(datetime_parser('1970-01-01') , testdate)
self.assertEqual(datetime_parser('1970-1-1'), testdate)
today = datetime.datetime.today()
beginthismonth = datetime.datetime(today.year, today.month, 1, 12, 1)
self.assertEqual(datetime_parser('BEGINTHISMONTH 12:1') , beginthismonth)
endapril = datetime.datetime(today.year, 4, 30, 12, 1, 1, 1)
self.assertEqual(datetime_parser('ENDAPRIL 12:01:01.000001') , endapril)
def plot_historic_PR_ages(created_ats, closed_ats, repository):
"""Plot historic PR ages."""
day = min(created_ats)
days = []
ages, ages_all, ages_rev = [], [], []
while day <= datetime_parser('TODAY') + ONE_DAY:
days.append(day)
open_counts = [0]*(len(GROUPS)+1)
for idx in range(0, len(created_ats)):
if created_ats[idx] <= day and closed_ats[idx] > day:
for i, grp in enumerate(GROUPS + [ENDGROUP]):
if day - created_ats[idx] < grp:
open_counts[i] += 1
break
ages.append(open_counts[::-1])
ages_rev.append(open_counts[:])
open_counts.append(sum(open_counts))
ages_all.append(open_counts[::-1])
day += ONE_DAY
print("%s: open = %s" % (days[-1], open_counts[-1]))
days = pd.to_datetime(days)
pd_df_all = pd.DataFrame(ages_all, days, columns=['all']+GROUP_LABELS[::-1]).sort_index().fillna(method='ffill')
res = pd_df_all.plot(kind='area', stacked=False, title="open %s PRs, by age" % repository)
res.legend(ncol=len(GROUP_LABELS)+1, fontsize='small')
plt.savefig('%s_PR_stats_all' % repository)
res = pd_df_all.plot(kind='area', stacked=False, title="open %s PRs, by age (zoomed)" % repository)
res.set_ylim([0, 50])
res.legend(ncol=len(GROUP_LABELS)+1, fontsize='small')
plt.savefig('%s_PR_stats_all_zoomed' % repository)
pd_df = pd.DataFrame(ages, days, columns=GROUP_LABELS[::-1]).sort_index().fillna(method='ffill')
res = pd_df.plot(kind='area', stacked=True, title="open %s PRs, by age (stacked)" % repository)
res.legend(ncol=len(GROUP_LABELS), fontsize='small')
plt.savefig('%s_PR_stats_all_stacked' % repository)
pd_df_all_year = pd_df_all.loc[LAST_YEAR:]
res = pd_df_all_year.plot(kind='area', stacked=False, title="open %s PRs, by age (last year)" % repository)
res.legend(ncol=len(GROUP_LABELS)+1, fontsize='small')
plt.savefig('%s_PR_stats_year' % repository)
pd_df_all_month = pd_df_all.loc[LAST_MONTH:]
res = pd_df_all_month.plot(kind='area', stacked=False, title="open %s PRs, by age (last month)" % repository)
res.legend(ncol=len(GROUP_LABELS)+1, fontsize='small')
plt.savefig('%s_PR_stats_month' % repository)
pd_df = pd.DataFrame(ages_rev, days, columns=GROUP_LABELS).sort_index().fillna(method='ffill')
res = pd_df.plot(kind='area', stacked=True, title="open %s PRs, by age (stacked)" % repository)
res.legend(ncol=len(GROUP_LABELS), fontsize='small')
plt.savefig('%s_PR_stats_all_stacked_rev' % repository)
res = pd_df.plot(kind='area', stacked=True, title="open %s PRs, by age (stacked, zoomed)" % repository)
res.set_ylim([0, 50])
res.legend(ncol=len(GROUP_LABELS), fontsize='small')
plt.savefig('%s_PR_stats_all_stacked_rev_zoomed' % repository)
def plot_historic_PR_ages(created_ats, closed_ats, repository):
"""Plot historic PR ages."""
day = min(created_ats)
days = []
ages, ages_all, ages_rev = [], [], []
while day <= datetime_parser("TODAY"):
days.append(day)
open_counts = [0] * (len(GROUPS) + 1)
for idx in xrange(0, len(created_ats)):
if created_ats[idx] <= day and closed_ats[idx] > day:
for i, grp in enumerate(GROUPS + [ENDGROUP]):
if day - created_ats[idx] < grp:
open_counts[i] += 1
break
ages.append(open_counts[::-1])
ages_rev.append(open_counts[:])
open_counts.append(sum(open_counts))
ages_all.append(open_counts[::-1])
day += ONE_DAY
pd_df_all = pd.DataFrame(ages_all, days, columns=["all"] + GROUP_LABELS[::-1]).sort().fillna(method="ffill")
res = pd_df_all.plot(kind="area", stacked=False, title="open %s PRs, by age" % repository)
res.legend(ncol=len(GROUP_LABELS) + 1, fontsize="small")
plt.savefig("%s_PR_stats_all" % repository)
res = pd_df_all.plot(kind="area", stacked=False, title="open %s PRs, by age (zoomed)" % repository)
res.set_ylim([0, 50])
res.legend(ncol=len(GROUP_LABELS) + 1, fontsize="small")
plt.savefig("%s_PR_stats_all_zoomed" % repository)
pd_df = pd.DataFrame(ages, days, columns=GROUP_LABELS[::-1]).sort().fillna(method="ffill")
res = pd_df.plot(kind="area", stacked=True, title="open %s PRs, by age (stacked)" % repository)
res.legend(ncol=len(GROUP_LABELS), fontsize="small")
plt.savefig("%s_PR_stats_all_stacked" % repository)
pd_df_all_year = pd_df_all.select(lambda d: d > LAST_YEAR)
res = pd_df_all_year.plot(kind="area", stacked=False, title="open %s PRs, by age (last year)" % repository)
res.legend(ncol=len(GROUP_LABELS) + 1, fontsize="small")
plt.savefig("%s_PR_stats_year" % repository)
pd_df_all_month = pd_df_all.select(lambda d: d > LAST_MONTH)
res = pd_df_all_month.plot(kind="area", stacked=False, title="open %s PRs, by age (last month)" % repository)
res.legend(ncol=len(GROUP_LABELS) + 1, fontsize="small")
plt.savefig("%s_PR_stats_month" % repository)
pd_df = pd.DataFrame(ages_rev, days, columns=GROUP_LABELS).sort().fillna(method="ffill")
res = pd_df.plot(kind="area", stacked=True, title="open %s PRs, by age (stacked)" % repository)
res.legend(ncol=len(GROUP_LABELS), fontsize="small")
plt.savefig("%s_PR_stats_all_stacked_rev" % repository)
res = pd_df.plot(kind="area", stacked=True, title="open %s PRs, by age (stacked, zoomed)" % repository)
res.set_ylim([0, 50])
res.legend(ncol=len(GROUP_LABELS), fontsize="small")
plt.savefig("%s_PR_stats_all_stacked_rev_zoomed" % repository)
def main():
opts = {
"github-account": ("GitHub account where repository is located", None, "store", "hpcugent", "a"),
"github-user": ("GitHub user to use (for authenticated access)", None, "store", "boegel", "u"),
"repository": ("Repository to use", None, "store", "easybuild-easyconfigs", "r"),
}
go = simple_option(go_dict=opts, descr="Script to print overview of pull requests for a GitHub repository")
pickle_file = None
if go.args:
pickle_file = go.args[0]
prs = fetch_pr_data(pickle_file, go.options.github_user, go.options.github_account, go.options.repository)
created_ats = [datetime_parser(pr["created_at"].split("T")[0]) for pr in prs]
closed_ats = [datetime_parser((pr["closed_at"] or "T").split("T")[0] or "ENDNEXTMONTH") for pr in prs]
print("Plotting...")
plot_historic_PR_ages(created_ats, closed_ats, go.options.repository)
plot_open_closed_PRs(created_ats, closed_ats, go.options.repository)
def plot_prs_by_author(created_ats, authors, repository):
"""Plot overview of PR authors."""
day = min(created_ats)
days = []
uniq_authors = nub(authors)
print("Found %d unique PR authors for %s repository" % (len(uniq_authors), repository))
author_counts = []
while day <= datetime_parser('TODAY'):
days.append(day)
counts = [0] * len(uniq_authors)
for idx in range(0, len(created_ats)):
if created_ats[idx] <= day:
for i, author in enumerate(uniq_authors):
if authors[idx] == author:
counts[i] += 1
break
author_counts.append(counts)
day += ONE_DAY
# filter author counts, only show top 10 author, collapse remaining authors into 'other'
sorted_author_counts = sorted(enumerate(author_counts[-1]), key=lambda x: x[1], reverse=True)
top_idxs = [idx for (idx, _) in sorted_author_counts[:30]]
other_idxs = [idx for idx in range(len(uniq_authors)) if idx not in top_idxs]
plot_author_counts = []
for day_counts in author_counts:
plot_day_counts = []
for idx in top_idxs[::-1]:
plot_day_counts.append(day_counts[idx])
other_day_count = 0
for idx in other_idxs:
other_day_count += day_counts[idx]
plot_day_counts.append(other_day_count)
plot_author_counts.append(plot_day_counts)
plot_authors = []
for idx in top_idxs[::-1]:
plot_authors.append('%s (%d)' % (uniq_authors[idx], author_counts[-1][idx]))
plot_authors.append('OTHER (%d)' % plot_author_counts[-1][-1])
pd_df = pd.DataFrame(plot_author_counts, days, columns=plot_authors).sort_index().fillna(method='ffill')
res = pd_df.plot(kind='area', stacked=True, title="%s PRs by author (stacked)" % repository)
res.legend(ncol=2, fontsize='small', loc='best')
plt.savefig('%s_PR_per_author_stacked' % repository)
def take_action(self, action, dest, opt, value, values, parser):
"""extended take_action"""
orig_action = action # keep copy
if action == 'shorthelp':
parser.print_shorthelp()
parser.exit()
elif action in ('store_true', 'store_false', 'store_debuglog'):
if action == 'store_debuglog':
action = 'store_true'
if opt.startswith("--%s-" % self.ENABLE):
# keep action
pass
elif opt.startswith("--%s-" % self.DISABLE):
# reverse action
if action in ('store_true', 'store_debuglog'):
action = 'store_false'
elif action in ('store_false', ):
action = 'store_true'
if orig_action == 'store_debuglog' and action == 'store_true':
setLogLevelDebug()
Option.take_action(self, action, dest, opt, value, values, parser)
elif action in self.EXTOPTION_EXTRA_OPTIONS:
if action == "extend":
# comma separated list convert in list
lvalue = value.split(self.EXTEND_SEPARATOR)
values.ensure_value(dest, []).extend(lvalue)
elif action == "date":
lvalue = date_parser(value)
setattr(values, dest, lvalue)
elif action == "datetime":
lvalue = datetime_parser(value)
setattr(values, dest, lvalue)
else:
raise (Exception(
"Unknown extended option action %s (known: %s)" %
(action, self.EXTOPTION_EXTRA_OPTIONS)))
else:
Option.take_action(self, action, dest, opt, value, values, parser)
# set flag to mark as passed by action (ie not by default)
# - distinguish from setting default value through option
if hasattr(values, '_action_taken'):
values._action_taken[dest] = True
def take_action(self, action, dest, opt, value, values, parser):
"""Extended take_action"""
orig_action = action # keep copy
if action == 'shorthelp':
parser.print_shorthelp()
parser.exit()
elif action in ('store_true', 'store_false',) + self.EXTOPTION_LOG:
if action in self.EXTOPTION_LOG:
action = 'store_true'
if opt.startswith("--%s-" % self.ENABLE):
# keep action
pass
elif opt.startswith("--%s-" % self.DISABLE):
# reverse action
if action in ('store_true',) + self.EXTOPTION_LOG:
action = 'store_false'
elif action in ('store_false',):
action = 'store_true'
if orig_action in('store_debuglog', 'store_infolog', 'store_warninglog') and action == 'store_true':
setLogLevel(orig_action.split('_')[1][:-3].upper())
Option.take_action(self, action, dest, opt, value, values, parser)
elif action in self.EXTOPTION_EXTRA_OPTIONS:
if action == "extend":
# comma separated list convert in list
lvalue = value.split(self.EXTEND_SEPARATOR)
values.ensure_value(dest, []).extend(lvalue)
elif action == "date":
lvalue = date_parser(value)
setattr(values, dest, lvalue)
elif action == "datetime":
lvalue = datetime_parser(value)
setattr(values, dest, lvalue)
else:
raise(Exception("Unknown extended option action %s (known: %s)" %
(action, self.EXTOPTION_EXTRA_OPTIONS)))
else:
Option.take_action(self, action, dest, opt, value, values, parser)
# set flag to mark as passed by action (ie not by default)
# - distinguish from setting default value through option
if hasattr(values, '_action_taken'):
values._action_taken[dest] = True
def test_datetime_parser(self):
"""Test the date_parser"""
testdate = datetime.datetime(1970, 1, 1)
self.assertEqual(datetime_parser('1970-01-01'), testdate)
self.assertEqual(datetime_parser('1970-1-1'), testdate)
today = datetime.datetime.today()
todaytime = datetime.datetime(today.year, today.month, today.day)
oneday = datetime.timedelta(days=1)
self.assertEqual(datetime_parser('TODAY'), todaytime)
self.assertEqual(datetime_parser('YESTERDAY'), todaytime - oneday)
self.assertEqual(datetime_parser('TOMORROW'), todaytime + oneday)
beginthismonth = datetime.datetime(today.year, today.month, 1, 12, 1)
self.assertEqual(datetime_parser('BEGINTHISMONTH 12:1'),
beginthismonth)
todayend = todaytime
self.assertEqual(datetime_parser('TODAY BEGIN'), todaytime)
self.assertEqual(datetime_parser('TODAY END'),
todaytime + oneday - datetime.timedelta(seconds=1))
self.assertEqual(datetime_parser('YESTERDAY END'),
todaytime - datetime.timedelta(seconds=1))
def plot_open_closed_PRs(created_ats, closed_ats, repository):
"""Plot open/closed PRs (in total) per day."""
opened_closed = []
days = []
day = min(created_ats)
while day <= datetime_parser('TODAY'):
days.append(day)
opened = sum([d.date() == day.date() for d in created_ats])
closed = sum([d.date() == day.date() for d in closed_ats])
open_cnt = opened - closed
opened_closed.append((open_cnt, opened, closed))
day += ONE_DAY
pd_df = pd.DataFrame(opened_closed, days, columns=['open', 'opened', 'closed']).sort_index()
res = pd_df.plot(kind='bar', title="open/opened/closed PRs")
res.legend(loc='upper left', ncol=3, fontsize='small')
plt.savefig('%s_opened_closed_PRs' % repository)
pd_df_month = pd_df.loc[LAST_MONTH:]
res = pd_df_month.plot(kind='bar', title="open/opened/closed PRs (last month)")
res.legend(loc='upper left', ncol=3, fontsize='small')
plt.savefig('%s_opened_closed_PRs_month' % repository)
pd_df_total = pd_df.cumsum()
res = pd_df_total.plot(kind='line', title="total open/opened/closed %s PRs" % repository)
res.legend(loc='upper left', ncol=3, fontsize='small')
plt.savefig('%s_opened_closed_PRs_cumulative' % repository)
pd_df_total_year = pd_df_total.loc[LAST_YEAR:]
res = pd_df_total_year.plot(kind='line', title="total open/opened/closed %s PRs (last year)" % repository)
res.legend(loc='upper left', ncol=3, fontsize='small')
plt.savefig('%s_opened_closed_PRs_cumulative_year' % repository)
pd_df_total_month = pd_df_total.loc[LAST_MONTH:]
res = pd_df_total_month.plot(kind='line', title="total open/opened/closed %s PRs (last month)" % repository)
res.legend(loc='upper left', ncol=3, fontsize='small')
plt.savefig('%s_opened_closed_PRs_cumulative_month' % repository)
def plot_open_closed_PRs(created_ats, closed_ats, repository):
"""Plot open/closed PRs (in total) per day."""
opened_closed = []
days = []
day = min(created_ats)
while day <= datetime_parser("TODAY"):
days.append(day)
opened = sum([d.date() == day.date() for d in created_ats])
closed = sum([d.date() == day.date() for d in closed_ats])
open_cnt = opened - closed
opened_closed.append((open_cnt, opened, closed))
day += ONE_DAY
pd_df = pd.DataFrame(opened_closed, days, columns=["open", "opened", "closed"]).sort()
res = pd_df.plot(kind="bar", title="open/opened/closed PRs")
res.legend(loc="upper left", ncol=3, fontsize="small")
plt.savefig("%s_opened_closed_PRs" % repository)
pd_df_month = pd_df.select(lambda d: d > LAST_MONTH)
res = pd_df_month.plot(kind="bar", title="open/opened/closed PRs (last month)")
res.legend(loc="upper left", ncol=3, fontsize="small")
plt.savefig("%s_opened_closed_PRs_month" % repository)
pd_df_total = pd_df.cumsum()
res = pd_df_total.plot(kind="line", title="total open/opened/closed %s PRs" % repository)
res.legend(loc="upper left", ncol=3, fontsize="small")
plt.savefig("%s_opened_closed_PRs_cumulative" % repository)
pd_df_total_year = pd_df_total.select(lambda d: d > LAST_YEAR)
res = pd_df_total_year.plot(kind="line", title="total open/opened/closed %s PRs (last year)" % repository)
res.legend(loc="upper left", ncol=3, fontsize="small")
plt.savefig("%s_opened_closed_PRs_cumulative_year" % repository)
pd_df_total_month = pd_df_total.select(lambda d: d > LAST_MONTH)
res = pd_df_total_month.plot(kind="line", title="total open/opened/closed %s PRs (last month)" % repository)
res.legend(loc="upper left", ncol=3, fontsize="small")
plt.savefig("%s_opened_closed_PRs_cumulative_month" % repository)
import easybuild.tools.build_log # required to obtain an EasyBuild logger
from easybuild.tools.github import GITHUB_API_URL, fetch_github_token
from pr_overview import fetch_prs_data
GROUPS = [
datetime.timedelta(7, 0, 0), # 1 week
datetime.timedelta(14, 0, 0), # 2 weeks
datetime.timedelta(30, 0, 0), # ~1 month
datetime.timedelta(60, 0, 0), # ~2 months
datetime.timedelta(180, 0, 0), # ~6 months
]
ENDGROUP = datetime.timedelta(10 ** 6, 0, 0) # very large
GROUP_LABELS = ["<1w", "1-2w", "2w-1m", "1-2m", "2-6m", ">6m"]
LAST_MONTH = datetime_parser("TODAY") - datetime.timedelta(30, 0, 0)
LAST_YEAR = datetime_parser("TODAY") - datetime.timedelta(365, 0, 0)
PICKLE_FILE = "%s_prs.dat"
ONE_DAY = datetime.timedelta(1, 0, 0)
def fetch_pr_data(pickle_file, github_user, github_account, repository):
"""Fetch PR data; either download or load from pickle file."""
if pickle_file:
print("Loading PR data from %s" % pickle_file)
prs = cPickle.load(open(pickle_file, "r"))
else:
github_token = fetch_github_token(github_user)
github = RestClient(GITHUB_API_URL, username=github_user, token=github_token, user_agent="eb-pr-stats")
gh_repo = github.repos[github_account][repository]
def gen_table_rows(prs):
"""Generate table rows."""
col_tmpls = [
"{v: %(number)s, f: '<a href=\"%(html_url)s\">#%(number)s</a>'}",
"'%(user_login)s'",
"'%(title)s'",
"{v: %(age)s, f: '%(age)s days'}",
"%(last_update)s",
"%(unit_test)s",
"%(style_check)s",
"%(test_reports)s",
"%(status)s",
"%(signed_off)s",
"%(participants)s", # FIXME
"0", # FIXME
]
row_tmpl = " [" + ', '.join(col_tmpls) + "],"
example_ut = [-1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
example_sc = [-1, -1, -1, 0, 0, 0, 1, 1, 1, -1, -1, -1, 0, 0, 0, 1, 1, 1, -1, -1, -1, 0, 0, 0, 1, 1, 1]
example_tr = [-1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1]
example_statuses = [-3, -2, -1, -2, -1, 0, -1, 0, 1, -2, -1, 0, -1, 0, 1, 0, 1, 2, -1, 0, 1, 0, 1, 2, 1, 2, 3]
merged_today = 0
todays_date = date_parser('TODAY')
last_update = datetime_parser(prs[0]['updated_at'].replace('T', ' ')[:-1])
lines = [' data.addRows([']
for pr in prs:
if pr['closed_at']:
if date_parser(pr['closed_at'].split('T')[0]) == todays_date:
merged_today += 1
# only consider open PRs
if pr['state'] != 'open':
continue
# shorten long titles, escape single quotes
if len(pr['title']) > 30:
pr['title'] = pr['title'][:30] + '...'
pr['title'] = pr['title'].replace("'", "\\'")
# determine status based on results of unit tests, style check and test reports
unit_test = TEST_VALUES_MAP[pr['combined_status']]
# initial value: '???' (unknown)
style_check = 0
test_reports = 0
signed_off = False
# iterate over all comments, last hit wins
for comment in pr['issue_comments']['bodies']:
if 'lgtm' in comment.lower() or 'style review ok' in comment.lower():
style_check = 1
if comment.lower().startswith('test report'):
if 'SUCCESS' in comment:
test_reports = 1
elif 'FAILED' in comment:
test_reports = -1
if 'good to go' in comment.lower():
signed_off = True
status = unit_test + style_check + test_reports
participants = len(nub(pr['issue_comments']['users']))
year, month, day, hour, minutes, seconds = pr['updated_at'].replace('T', '@')[:-1].replace(':', '@').replace('-', '@').split('@')
pr.update({
'age': (datetime_parser('TODAY') - datetime_parser(pr['created_at'].split('T')[0])).days,
'last_update': "new Date(%s, %s, %s, %s, %s, %s)" % (year, month, day, hour, minutes, seconds),
'participants': participants,
'signed_off': ['', 'true'][signed_off],
'status': status,
'style_check': TEST_RESULTS[style_check+1],
'test_reports': TEST_RESULTS[test_reports+1],
'unit_test': TEST_RESULTS[unit_test+1],
'user_login': pr['user']['login'],
})
lines.append(row_tmpl % pr)
last_update = max(datetime_parser(pr['updated_at'].replace('T', ' ')[:-1]), last_update)
lines.append(' ]);')
return len(lines)-2, '\n'.join(lines), merged_today, last_update
def gen_table_rows(prs):
"""Generate table rows."""
col_tmpls = [
"{v: %(number)s, f: '<a href=\"%(html_url)s\">#%(number)s</a>'}",
"'%(user_login)s'",
"'%(title)s'",
"{v: %(age)s, f: '%(age)s days'}",
"%(last_update)s",
"%(unit_test)s",
"%(style_check)s",
"%(test_reports)s",
"%(status)s",
"%(signed_off)s",
"%(participants)s", # FIXME
"0", # FIXME
]
row_tmpl = " [" + ', '.join(col_tmpls) + "],"
merged_today = 0
todays_date = date_parser('TODAY')
last_update = datetime_parser(prs[0]['updated_at'].replace('T', ' ')[:-1])
lines = [' data.addRows([']
for pr in prs:
if pr['closed_at']:
if date_parser(pr['closed_at'].split('T')[0]) == todays_date:
merged_today += 1
# only consider open PRs
if pr['state'] != 'open':
continue
# shorten long titles, escape single quotes
if len(pr['title']) > 30:
pr['title'] = pr['title'][:30] + '...'
pr['title'] = pr['title'].replace("'", "\\'")
# determine status based on results of unit tests, style check and test reports
unit_test = TEST_VALUES_MAP[pr['combined_status']]
# initial value: '???' (unknown)
style_check = 0
test_reports = 0
signed_off = False
# iterate over all comments, last hit wins
for comment in pr['issue_comments']['bodies']:
if 'lgtm' in comment.lower() or 'style review ok' in comment.lower():
style_check = 1
if comment.lower().startswith('test report'):
if 'SUCCESS' in comment:
test_reports = 1
elif 'FAILED' in comment:
test_reports = -1
if 'good to go' in comment.lower():
signed_off = True
status = unit_test + style_check + test_reports
participants = len(nub(pr['issue_comments']['users']))
raw_updated_at = pr['updated_at'].replace('T', '@')[:-1].replace(':', '@').replace('-', '@')
year, month, day, hour, minutes, seconds = raw_updated_at.split('@')
pr.update({
'age': (datetime_parser('TODAY') - datetime_parser(pr['created_at'].split('T')[0])).days,
'last_update': "new Date(%s, %s, %s, %s, %s, %s)" % (year, month, day, hour, minutes, seconds),
'participants': participants,
'signed_off': ['', 'true'][signed_off],
'status': status,
'style_check': TEST_RESULTS[style_check+1],
'test_reports': TEST_RESULTS[test_reports+1],
'unit_test': TEST_RESULTS[unit_test+1],
'user_login': pr['user']['login'],
})
lines.append(row_tmpl % pr)
last_update = max(datetime_parser(pr['updated_at'].replace('T', ' ')[:-1]), last_update)
lines.append(' ]);')
return len(lines)-2, '\n'.join(lines), merged_today, last_update
from easybuild.tools.utilities import nub
from vsc.utils.dateandtime import date_parser, datetime_parser
log = fancylogger.getLogger()
GROUPS = [
datetime.timedelta(7, 0, 0), # 1 week
datetime.timedelta(14, 0, 0), # 2 weeks
datetime.timedelta(30, 0, 0), # ~1 month
datetime.timedelta(60, 0, 0), # ~2 months
datetime.timedelta(180, 0, 0), # ~6 months
]
ENDGROUP = datetime.timedelta(10**6, 0, 0) # very large
GROUP_LABELS = ['<1w', '1-2w', '2w-1m', '1-2m', '2-6m', '>6m']
LAST_MONTH = datetime_parser('TODAY') - datetime.timedelta(30, 0, 0)
LAST_YEAR = datetime_parser('TODAY') - datetime.timedelta(365, 0, 0)
PICKLE_FILE = '%s_prs.dat'
ONE_DAY = datetime.timedelta(1, 0, 0)
HTML_FILE = '%s_pr_overview.html'
HTML_HEADER = """
<html>
<head>
<script type="text/javascript" src="https://www.google.com/jsapi"></script>
<script type="text/javascript">
google.load("visualization", "1", {packages:["table"]});
google.setOnLoadCallback(drawTable);
function drawTable() {
var data = new google.visualization.DataTable();
"""
