def releases_info(startdate, enddate, project, opts, people_out, affs_out):
# Releases information.
data = {}
filters = MetricFilters("month", startdate, enddate,
["project", str(project)], opts.npeople,
people_out, affs_out)
# SCM report
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
dataset = scm_report(scm_dbcon, filters)
data["scm"] = dataset
#ITS report
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho,
opts.dbidentities, opts.dbprojects)
dataset = its_report(its_dbcon, filters)
data["its"] = dataset
#SCR Report
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview,
opts.dbidentities, opts.dbprojects)
dataset = scr_report(scr_dbcon, filters)
data["scr"] = dataset
return data
def timezone_analysis(opts):
from vizgrimoire.analysis.timezone import Timezone
from vizgrimoire.SCM import SCM
from vizgrimoire.MLS import MLS
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats,
opts.dbidentities, opts.dbprojects)
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'" + releases[0] + "'"
enddate = "'" + releases[1] + "'"
filters = MetricFilters(period, startdate, enddate, [], 10, "", "")
tz = Timezone(scm_dbcon, filters)
dataset = tz.result(SCM)
labels = dataset["tz"]
commits = dataset["commits"]
authors = dataset["authors"]
bar_chart("Timezone git activity", labels, commits, "commits_tz", authors,
["commits", "authors"])
tz = Timezone(mls_dbcon, filters)
dataset = tz.result(MLS)
messages = dataset["messages"]
authors = dataset["authors"]
bar_chart("Timezone mailing list activity", labels, messages,
"messages_tz", authors, ["messages", "authors"])
def data_source_increment_activity(opts, people_out, affs_out):
# Per data source, the increment or decrement of the activity is displayed
dataset = {}
data_sources = ["Gits", "Tickets", "Mailing Lists", "Gerrit", "Askbot", "IRC"]
action = ["commits", "closed tickets", "sent emails", "submitted reviews", "posted questions", "messages"]
net_values = []
rel_values = [] #percentage wrt the previous 365 days
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities)
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities)
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats, opts.dbidentities)
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities)
qaforums_dbcon = QAForumsQuery(opts.dbuser, opts.dbpassword, opts.dbqaforums, opts.dbidentities)
irc_dbcon = IRCQuery(opts.dbuser, opts.dbpassword, opts.dbirc, opts.dbidentities)
period = "month"
type_analysis = None
releases = opts.releases.split(",")[-2:]
startdate = "'"+releases[0]+"'"
enddate = "'"+releases[1]+"'"
filters = MetricFilters(period, startdate, enddate, None, 10, people_out, affs_out)
commits = scm.Commits(scm_dbcon, filters)
closed = its.Closed(its_dbcon, filters)
emails = mls.EmailsSent(mls_dbcon, filters)
submitted = scr.Submitted(scr_dbcon, filters)
questions = qa.Questions(qaforums_dbcon, filters)
messages = irc.Sent(irc_dbcon, filters)
from vizgrimoire.ITS import ITS
ITS.set_backend("jira")
net_values.append(commits.get_trends(releases[1], 90)["commits_90"])
rel_values.append(commits.get_trends(releases[1], 90)["percentage_commits_90"])
net_values.append(closed.get_trends(releases[1], 90)["closed_90"])
rel_values.append(closed.get_trends(releases[1], 90)["percentage_closed_90"])
net_values.append(emails.get_trends(releases[1], 90)["sent_90"])
rel_values.append(emails.get_trends(releases[1], 90)["percentage_sent_90"])
net_values.append(submitted.get_trends(releases[1], 90)["submitted_90"])
rel_values.append(submitted.get_trends(releases[1], 90)["percentage_submitted_90"])
net_values.append(questions.get_trends(releases[1], 90)["qsent_90"])
rel_values.append(questions.get_trends(releases[1], 90)["percentage_qsent_90"])
net_values.append(messages.get_trends(releases[1], 90)["sent_90"])
rel_values.append(messages.get_trends(releases[1], 90)["percentage_sent_90"])
createCSV({"datasource":data_sources, "metricsnames":action, "relativevalues":rel_values, "netvalues":net_values}, "./release/data_source_evolution.csv")
def projects_efficiency(opts, people_out, affs_out):
# BMI and time to review in mean per general project
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities, opts.dbprojects)
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities, opts.dbprojects)
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities, opts.dbprojects)
bmi_list = []
time2review_list = []
bmi_its = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'"+releases[0]+"'"
enddate = "'"+releases[1]+"'"
type_analysis = None
project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10,
people_out, affs_out)
scr_bmi = scr.BMISCR(scr_dbcon, project_filters)
time2review = scr.TimeToReview(scr_dbcon, project_filters)
# ITS BMI index
from vizgrimoire.ITS import ITS
ITS.set_backend("jira")
opened = its.Opened(its_dbcon, project_filters)
closed = its.Closed(its_dbcon, project_filters)
tickets_opened = opened.get_agg()["opened"]
tickets_closed = closed.get_agg()["closed"]
its_bmi = 0
if tickets_closed > 0:
its_bmi = round(float(tickets_closed)/float(tickets_opened), 2)
bmi_list.append(round(scr_bmi.get_agg()["bmiscr"], 2))
time2review_list.append(round(time2review.get_agg()["review_time_days_median"], 2))
bmi_its.append(its_bmi)
createCSV({"projects":"general", "bmi":bmi_list, "timereview":time2review_list, "bmiits":bmi_its}, "./release/integrated_projects_efficiency.csv")
from vizgrimoire.datahandlers.data_handler import DHESA
# parse options
opts = read_options()
today = datetime.date.today()
startdate = today - datetime.timedelta(days=PERIOD)
startdate = "'" + startdate.strftime("%Y-%m-%d") + "'"
enddate = "'" + today.strftime("%Y-%m-%d") + "'"
# Projects analysis. This includes SCM, SCR and ITS.
people_out = []
affs_out = ["-Bot", "-Individual", "-Unknown"]
# Lines of code per repository
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities)
data = {}
# List of projects:
query = "select project_id, id from projects"
projects = scm_dbcon.ExecuteQuery(query)
for project in projects["id"]:
try:
#Obtaining ncloc metric from Sonar dataset
with open("../../../json/" + project +
"-metrics-sonarqube.json") as sonar_data:
sonar_metrics = json.load(sonar_data)
except:
import vizgrimoire.metrics.mls_metrics as mls
from vizgrimoire.GrimoireUtils import createJSON
from vizgrimoire.GrimoireSQL import SetDBChannel
# parse options
opts = read_options()
# obtain list of releases by tuples [(date1, date2), (date2, date3), ...]
releases = build_releases(opts.releases)
for release in releases:
startdate = "'" + release[0] + "'"
enddate = "'" + release[1] + "'"
filters = MetricFilters("month", startdate, enddate, None,
opts.npeople)
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities)
#SCM report
print("\n* SCM summary")
scm_report(scm_dbcon, filters)
#QAForums report
print("\n* Askbot summary")
print(opts.dbuser, opts.dbpassword, opts.dbqaforums, opts.dbidentities)
qa_dbcon = QAForumsQuery(opts.dbuser, opts.dbpassword, opts.dbqaforums,
opts.dbidentities)
qaforums_report(qa_dbcon, filters)
#MLS Report
print("\n* Mailing list summary")
print(opts.dbuser, opts.dbpassword, opts.dbmlstats, opts.dbidentities)
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats,
def general_info(opts, releases, people_out, affs_out):
# General info from MLS, IRC and QAForums.
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
# analysis currently failing
#timezone_analysis(opts)
core = []
regular = []
occasional = []
authors_month = []
emails = []
emails_senders = []
emails_senders_init = []
questions = []
answers = []
comments = []
qsenders = []
irc_sent = []
irc_senders = []
releases_data = {}
for release in releases:
startdate = "'" + release[0] + "'"
enddate = "'" + release[1] + "'"
print "General info per release: " + startdate + " - " + enddate
filters = MetricFilters("month", startdate, enddate, None,
opts.npeople, people_out, affs_out)
# SCM info
print " General info: SCM"
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
dataset = scm_general(scm_dbcon, filters)
core.append(dataset["core"])
regular.append(dataset["regular"])
occasional.append(dataset["occasional"])
authors_month.append(float(dataset["authorsperiod"]))
top_authors = dataset["topauthors"]
release_pos = releases.index(release)
createCSV(top_authors,
"./release/top_authors_release" + str(release_pos) + ".csv")
# MLS info
print " General info: MLS"
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats,
opts.dbidentities, opts.dbprojects)
dataset = mls_report(mls_dbcon, filters)
emails.append(dataset["sent"])
emails_senders.append(dataset["senders"])
emails_senders_init.append(dataset["senders_init"])
# QAForums info
print " General info: QAForums"
qaforums_dbcon = QAForumsQuery(opts.dbuser, opts.dbpassword,
opts.dbqaforums, opts.dbidentities,
opts.dbprojects)
dataset = qaforums_report(qaforums_dbcon, filters)
questions.append(dataset["questions"])
answers.append(dataset["answers"])
comments.append(dataset["comments"])
qsenders.append(dataset["qsenders"])
# IRC info
print " General info: IRC"
irc_dbcon = IRCQuery(opts.dbuser, opts.dbpassword, opts.dbirc,
opts.dbidentities, opts.dbprojects)
dataset = irc_report(irc_dbcon, filters)
irc_sent.append(dataset["sent"])
irc_senders.append(dataset["senders"])
labels = [
"13-Q3", "13-Q4", "14-Q1", "14-Q2", "14-Q3", "14-Q4", "15-Q1", "15-Q2"
]
#labels = ["2013-Q3", "2013-Q4", "2014-Q1", "2014-Q2"]
bar_chart("Emails sent", labels, emails, "emails")
createCSV({"labels": labels, "emails": emails}, "./release/emails.csv")
bar_chart("People sending emails", labels, emails_senders,
"emails_senders")
createCSV({
"labels": labels,
"senders": emails_senders
}, "./release/emails_senders.csv")
bar_chart("People initiating threads", labels, emails_senders_init,
"emails_senders_init")
createCSV({
"labels": labels,
"senders": emails_senders_init
}, "./release/emails_senders_init.csv")
bar_chart("Questions", labels, questions, "questions")
createCSV({
"labels": labels,
"questions": questions
}, "./release/questions.csv")
bar_chart("Answers", labels, answers, "answers")
createCSV({"labels": labels, "answers": answers}, "./release/answers.csv")
bar_chart("Comments", labels, comments, "comments")
createCSV({
"labels": labels,
"comments": comments
}, "./release/comments.csv")
bar_chart("People asking Questions", labels, qsenders, "question_senders")
createCSV({
"labels": labels,
"senders": qsenders
}, "./release/question_senders.csv")
bar_chart("Messages in IRC channels", labels, irc_sent, "irc_sent")
createCSV({
"labels": labels,
"messages": irc_sent
}, "./release/irc_sent.csv")
bar_chart("People in IRC channels", labels, irc_senders, "irc_senders")
createCSV({
"labels": labels,
"senders": irc_senders
}, "./release/irc_senders.csv")
bar3_chart("Community structure", labels, regular, "onion", core,
occasional, ["casual", "regular", "core"])
createCSV(
{
"labels": labels,
"core": core,
"regular": regular,
"occasional": occasional
}, "./release/onion_model.csv")
bar_chart("Developers per month", labels, authors_month, "authors_month")
createCSV({
"labels": labels,
"authormonth": authors_month
}, "./release/authors_month.csv")
# other analysis
print "Other analysis"
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities)
def projects_efficiency(opts, people_out, affs_out):
# BMI and time to review in mean per general project
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview,
opts.dbidentities, opts.dbprojects)
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho,
opts.dbidentities, opts.dbprojects)
projects = integrated_projects(scm_dbcon)
projects_ids = projects["subproject_id"]
projects_list = []
bmi_list = []
time2review_list = []
bmi_its = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'" + releases[0] + "'"
enddate = "'" + releases[1] + "'"
for project_id in projects_ids:
project_title = "'" + project_id + "'"
type_analysis = ["project", project_id]
project_filters = MetricFilters(period, startdate, enddate,
type_analysis, 10, people_out,
affs_out)
scr_bmi = scr.BMISCR(scr_dbcon, project_filters)
time2review = scr.TimeToReview(scr_dbcon, project_filters)
# ITS BMI index
from vizgrimoire.ITS import ITS
ITS.set_backend("launchpad")
if project_id == 'Documentation':
ITS._get_backend(
).closed_condition = "(new_value='Fix Committed' or new_value='Fix Released')"
else:
ITS.closed_condition = "(new_value='Fix Committed')"
opened = its.Opened(its_dbcon, project_filters)
closed = its.Closed(its_dbcon, project_filters)
tickets_opened = opened.get_agg()["opened"]
tickets_closed = closed.get_agg()["closed"]
its_bmi = 0
if tickets_closed > 0:
its_bmi = round(float(tickets_closed) / float(tickets_opened), 2)
projects_list.append(project_id)
bmi_list.append(round(scr_bmi.get_agg()["bmiscr"], 2))
time2review_list.append(
round(time2review.get_agg()["review_time_days_median"], 2))
bmi_its.append(its_bmi)
createCSV(
{
"projects": projects_list,
"bmi": bmi_list,
"timereview": time2review_list,
"bmiits": bmi_its
}, "./release/integrated_projects_efficiency.csv")
## Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. from vizgrimoire.analysis.timezone import Timezone from vizgrimoire.SCM import SCM from vizgrimoire.MLS import MLS from vizgrimoire.metrics.query_builder import SCMQuery from vizgrimoire.metrics.query_builder import MLSQuery from vizgrimoire.metrics.metrics_filter import MetricFilters dbuser = "******" dbpassword = "" dbcvsanaly = "eclipse_source_code" dbmlstats = "eclipse_mailing_list" dbidentities = "eclipse_source_code" period = "month" startdate = "'2013-10-16'" enddate = "'2014-10-17'" scm_dbcon = SCMQuery(dbuser, dbpassword, dbcvsanaly, dbidentities) mls_dbcon = MLSQuery(dbuser, dbpassword, dbmlstats, dbidentities) filters = MetricFilters(period, startdate, enddate, [], 10, "", "") tz = Timezone(scm_dbcon, filters) dataset = tz.result(SCM) print dataset tz = Timezone(mls_dbcon, filters) dataset = tz.result(MLS) print dataset
fields = "SELECT COUNT(DISTINCT(s.id)) as commits, u.id, u.identifier as authors "
fields += "FROM actions a, scmlog s, people_uidentities pup, upeople u "
q = fields + projects_from + projects_where
q += " AND pup.people_id = s.author_id AND u.id = pup.uuid "
q += " AND a.commit_id = s.id "
q += " AND s.author_date >= " + self.filters.startdate + " and s.author_date < " + self.filters.enddate
q += " GROUP by u.id ORDER BY commits DESC, u.id"
q += " limit " + str(self.filters.npeople)
res = self.db.ExecuteQuery(q)
return res
if __name__ == '__main__':
#example using this class
filters = MetricFilters("week", "'2014-01-01'", "'2014-04-01'",
["project", "integrated"])
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259",
"dic_cvsanaly_openstack_2259")
top_authors = TopAuthorsProjects(dbcon, filters)
print top_authors.result()
#example using query_builder function
from vizgrimoire.metrics.query_builder import SCMQuery
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259",
"dic_cvsanaly_openstack_2259")
print dbcon.get_project_top_authors("integrated", "'2014-01-01'",
"'2014-04-01'", 10)
projects_where = " WHERE " + self.db.GetSQLProjectWhere(project)[3:]
fields = "SELECT COUNT(DISTINCT(s.id)) as company_commits, org.name as organizations "
fields += "FROM actions a, scmlog s, people_uidentities pup, upeople u, enrollments enr, organizations org "
q = fields + projects_from + projects_where
q += " AND pup.people_id = s.author_id AND u.id = pup.uuid "
q += " AND u.id = enr.upeople_id AND org.id = enr.organization_id "
q += " AND s.date >= enr.start and s.date < enr.end "
q += " AND s.date>=" + self.filters.startdate + " and s.date < " + self.filters.enddate
q += " AND a.commit_id = s.id "
q += " GROUP by org.name ORDER BY company_commits DESC, org.name"
q += " limit " + str(self.filters.npeople)
res = self.db.ExecuteQuery(q)
return res
if __name__ == '__main__':
#example using this class
filters = MetricFilters("week", "'2014-01-01'", "'2014-04-01'", ["project", "integrated"])
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259", "dic_cvsanaly_openstack_2259")
top_orgs = TopCompaniesProjects(dbcon, filters)
print top_orgs.result()
#example using query_builder function
from vizgrimoire.metrics.query_builder import SCMQuery
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259", "dic_cvsanaly_openstack_2259")
print dbcon.get_project_top_organizations("integrated", "'2014-01-01'", "'2014-04-01'", 10)
# Remove first and
projects_where = " WHERE " + self.db.GetSQLProjectWhere(project)[3:]
fields = "SELECT COUNT(DISTINCT(s.id)) as commits, u.id, u.identifier as authors "
fields += "FROM actions a, scmlog s, people_uidentities pup, upeople u "
q = fields + projects_from + projects_where
q += " AND pup.people_id = s.author_id AND u.id = pup.uuid "
q += " AND a.commit_id = s.id "
q += " AND s.author_date >= " + self.filters.startdate + " and s.author_date < " + self.filters.enddate
q += " GROUP by u.id ORDER BY commits DESC, u.id"
q += " limit " + str(self.filters.npeople)
res = self.db.ExecuteQuery(q)
return res
if __name__ == '__main__':
#example using this class
filters = MetricFilters("week", "'2014-01-01'", "'2014-04-01'", ["project", "integrated"])
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259", "dic_cvsanaly_openstack_2259")
top_authors = TopAuthorsProjects(dbcon, filters)
print top_authors.result()
#example using query_builder function
from vizgrimoire.metrics.query_builder import SCMQuery
dbcon = SCMQuery("root", "", "dic_cvsanaly_openstack_2259", "dic_cvsanaly_openstack_2259")
print dbcon.get_project_top_authors("integrated", "'2014-01-01'", "'2014-04-01'", 10)
name = "Projects"
desc = "Projects in the source code management system"
data_source = SCM
def get_list(self):
# Just get commits per project
startdate = self.filters.startdate
enddate = self.filters.enddate
type_analysis = ['project', None]
period = None
evol = False
mcommits = Commits(self.db, self.filters)
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
mfilter_orig = mcommits.filters
mcommits.filters = mfilter
commits = mcommits.get_agg()
mcommits.filters = mfilter_orig
checkListArray(commits)
return commits
if __name__ == '__main__':
filters1 = MetricFilters("month", "'2014-04-01'", "'2015-01-01'",
['repository', "'OpenID'"])
dbcon = SCMQuery("root", "", "cp_cvsanaly_GrimoireLibTests",
"cp_sortinghat_GrimoireLibTests")
gone = GoneAuthors(dbcon, filters1)
print gone.get_agg()
print gone.get_list()
actors_commits = np.array(actors['commits'])
percentage_commits = (actors_commits /
float(total_commits['commits'])) * 100
actors['percentage'] = list(percentage_commits)
return actors
def result(self, data_source=None, destdir=None):
if self.typeof_leader is None: return None
return self._top_actors()
# Examples of use
if __name__ == '__main__':
filters = MetricFilters("week", "'2010-01-01'", "'2014-01-01'",
["repository", "'nova.git'"], 10)
dbcon = SCMQuery(
"root",
"",
"dic_cvsanaly_openstack_2259",
"dic_cvsanaly_openstack_2259",
)
leaders = SCMLeaders(dbcon, filters, SCMLeaders.TYPEOF_LEADER_DEVELOPERS,
"'nova.git'", 180)
print leaders.result()
leaders = SCMLeaders(dbcon, filters, SCMLeaders.TYPEOF_LEADER_ORGS,
"'nova.git'", 180)
print leaders.result()
