def scm_report(dbcon, filters, output_dir):
# Basic activity and community metrics in source code
# management systems
dataset = {}
from vizgrimoire.analysis.onion_model import CommunityStructure
onion = CommunityStructure(dbcon, filters)
result = onion.result()
dataset["scm_core"] = result["core"]
dataset["scm_regular"] = result["regular"]
dataset["scm_occasional"] = result["occasional"]
authors_period = scm.AuthorsPeriod(dbcon, filters)
dataset["scm_authorsperiod"] = float(
authors_period.get_agg()["avg_authors_month"])
authors = scm.Authors(dbcon, filters)
top_authors = authors.get_list()
createJSON(top_authors, output_dir + "scm_top_authors.json")
createCSV(top_authors, output_dir + "scm_top_authors.csv")
commits = scm.Commits(dbcon, filters)
dataset["scm_commits"] = commits.get_agg()["commits"]
authors = scm.Authors(dbcon, filters)
dataset["scm_authors"] = authors.get_agg()["authors"]
#companies = scm.Companies(dbcon, filters)
#top_companies = companies.get_list(filters)
#createJSON()
#createCSV()
return dataset
def mls_report(dbcon, filters, output_dir):
dataset = {}
emails = mls.EmailsSent(dbcon, filters)
dataset["mls_sent"] = emails.get_agg()["sent"]
senders = mls.EmailsSenders(dbcon, filters)
dataset["mls_senders"] = senders.get_agg()["senders"]
senders_init = mls.SendersInit(dbcon, filters)
dataset["mls_senders_init"] = senders_init.get_agg()["senders_init"]
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
threads = Threads(filters.startdate, filters.enddate, dbcon.identities_db)
top_longest_threads = threads.topLongestThread(10)
top_longest_threads = serialize_threads(top_longest_threads, False,
threads)
createJSON(top_longest_threads,
output_dir + "/mls_top_longest_threads.json")
createCSV(top_longest_threads, output_dir + "/mls_top_longest_threads.csv")
top_crowded_threads = threads.topCrowdedThread(10)
top_crowded_threads = serialize_threads(top_crowded_threads, True, threads)
createJSON(top_crowded_threads,
output_dir + "/mls_top_crowded_threads.json")
createCSV(top_crowded_threads, output_dir + "/mls_top_crowded_threads.csv")
return dataset
def create_filter_report_all(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
if filter_name in ["people2","company", "country","repository","domain","project",
"company"+MetricFilters.DELIMITER+"country",
"company"+MetricFilters.DELIMITER+"project"] :
filter_all = Filter(filter_name, None)
agg_all = cls.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(cls()))
createJSON(agg_all, fn)
ITS.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = cls.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(cls()))
createJSON(evol_all, fn)
ITS.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
cls._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
cls._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def its_report(dbcon, filters):
# Per release its information
from vizgrimoire.ITS import ITS
ITS.set_backend("launchpad")
project_name = filters.type_analysis[1]
project_name = project_name.replace(" ", "")
if project_name == 'Documentation':
ITS._get_backend(
).closed_condition = "(new_value='Fix Committed' or new_value='Fix Released')"
else:
ITS.closed_condition = "(new_value='Fix Committed')"
opened = its.Opened(dbcon, filters)
createJSON(opened.get_agg(),
"./release/its_opened_" + project_name + ".json")
closed = its.Closed(dbcon, filters)
createJSON(closed.get_agg(),
"./release/its_closed_" + project_name + ".json")
dataset = {}
dataset["opened"] = opened.get_agg()["opened"]
dataset["closed"] = closed.get_agg()["closed"]
return dataset
def create_filter_report_all(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
check = False # activate to debug pull_requests
filter_name = filter_.get_name()
if filter_name == "people2" or filter_name == "company+country":
filter_all = Filter(filter_name, None)
agg_all = Pullpo.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_static_filename_all(Pullpo()))
createJSON(agg_all, fn)
evol_all = Pullpo.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_evolutionary_filename_all(Pullpo()))
createJSON(evol_all, fn)
if check:
Pullpo._check_report_all_data(evol_all, filter_, startdate,
enddate, identities_db, True,
period)
Pullpo._check_report_all_data(agg_all, filter_, startdate,
enddate, identities_db, False,
period)
else:
logging.error(filter_name +
" does not support yet group by items sql queries")
def create_filter_report_all(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
if filter_name in ["people2","company", "country","repository","domain","project",
"company"+MetricFilters.DELIMITER+"country",
"company"+MetricFilters.DELIMITER+"project"] :
filter_all = Filter(filter_name, None)
agg_all = cls.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(cls()))
createJSON(agg_all, fn)
ITS.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = cls.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(cls()))
createJSON(evol_all, fn)
ITS.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
cls._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
cls._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def scm_report(dbcon, filters, output_dir):
# Basic activity and community metrics in source code
# management systems
dataset = {}
from vizgrimoire.analysis.onion_model import CommunityStructure
onion = CommunityStructure(dbcon, filters)
result = onion.result()
dataset["scm_core"] = result["core"]
dataset["scm_regular"] = result["regular"]
dataset["scm_occasional"] = result["occasional"]
authors_period = scm.AuthorsPeriod(dbcon, filters)
dataset["scm_authorsperiod"] = float(authors_period.get_agg()["avg_authors_month"])
authors = scm.Authors(dbcon, filters)
top_authors = authors.get_list()
createJSON(top_authors, output_dir + "scm_top_authors.json")
createCSV(top_authors, output_dir + "scm_top_authors.csv")
commits = scm.Commits(dbcon, filters)
dataset["scm_commits"] = commits.get_agg()["commits"]
authors = scm.Authors(dbcon, filters)
dataset["scm_authors"] = authors.get_agg()["authors"]
#companies = scm.Companies(dbcon, filters)
#top_companies = companies.get_list(filters)
#createJSON()
#createCSV()
return dataset
def mls_report(dbcon, filters, output_dir):
dataset = {}
emails = mls.EmailsSent(dbcon, filters)
dataset["mls_sent"] = emails.get_agg()["sent"]
senders = mls.EmailsSenders(dbcon, filters)
dataset["mls_senders"] = senders.get_agg()["senders"]
senders_init = mls.SendersInit(dbcon, filters)
dataset["mls_senders_init"] = senders_init.get_agg()["senders_init"]
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
threads = Threads(filters.startdate, filters.enddate, dbcon.identities_db)
top_longest_threads = threads.topLongestThread(10)
top_longest_threads = serialize_threads(top_longest_threads, False, threads)
createJSON(top_longest_threads, output_dir + "/mls_top_longest_threads.json")
createCSV(top_longest_threads, output_dir + "/mls_top_longest_threads.csv")
top_crowded_threads = threads.topCrowdedThread(10)
top_crowded_threads = serialize_threads(top_crowded_threads, True, threads)
createJSON(top_crowded_threads, output_dir + "/mls_top_crowded_threads.json")
createCSV(top_crowded_threads, output_dir + "/mls_top_crowded_threads.csv")
return dataset
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
# top by filter, not supported by group all queries
MLS.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
if filter_name in ["people2","company","repository","country","domain","project"] :
filter_all = Filter(filter_name, None)
agg_all = MLS.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(MLS()))
createJSON(agg_all, fn)
MLS.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = MLS.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(MLS()))
createJSON(evol_all, fn)
MLS.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
MLS._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
MLS._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items:
item = item.replace("'", "\\'")
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db,
filter_item, npeople, False)
createJSON(top_senders,
destdir + "/" + filter_item.get_top_filename(MLS()))
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("company","project","repository"):
top_mergers = SCR.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(SCR()))
createJSON(top_mergers, fn)
def create_people_identifiers(startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.GrimoireUtils import check_array_values
logging.info("Generating people identifiers")
people = get_top_report(startdate, enddate, npeople, identities_db, only_people=True);
people_ids = [] # upeople_ids which need identifiers
people_data = {} # identifiers for upeople_ids
for ds in Report.get_data_sources():
periods = [".",".last year",".last month"]
top_names = ds.get_top_metrics();
for period in periods:
for top_name in top_names:
if top_name+period in people[ds.get_name()]:
if 'id' in people[ds.get_name()][top_name+period]:
people_ids += check_array_values(people[ds.get_name()][top_name+period])['id']
people_ids = list(set(people_ids))
import vizgrimoire.People as People
for upeople_id in people_ids:
people_data[upeople_id] = People.GetPersonIdentifiers(identities_db, upeople_id)
all_top_min_ds = get_top_people(startdate, enddate, identities_db)
for upeople_id in all_top_min_ds:
people_data[upeople_id] = People.GetPersonIdentifiers(identities_db, upeople_id)
createJSON(people_data, destdir+"/people.json")
return people_ids
def result(self):
fields = Set([])
tables = Set([])
filters = Set([])
query = """select distinct(new_value) as states
from changes
where field = 'Status' """
states = self.db.ExecuteQuery(query)
data = {}
for state in states["states"]:
if state <> 'Resolved': continue
query = self._sql(state)
state_data = self.db.ExecuteQuery(query)
state_data = completePeriodIds(state_data, self.filters.period,
self.filters.startdate,
self.filters.enddate)
if not data:
data = state_data
data[state] = data["changes"]
data.pop("changes") # remove not needed data
else:
data[state] = state_data["changes"]
# TODO: Hardcoded creation of file
createJSON(data, "../../../../json/its-changes.json")
return data
def mls_report(dbcon, filters):
# Per release MLS information
emails = mls.EmailsSent(dbcon, filters)
createJSON(emails.get_agg(), "./release/mls_emailssent.json")
senders = mls.EmailsSenders(dbcon, filters)
createJSON(senders.get_agg(), "./release/mls_emailssenders.json")
senders_init = mls.SendersInit(dbcon, filters)
createJSON(senders_init.get_agg(), "./release/mls_sendersinit.json")
dataset = {}
dataset["sent"] = emails.get_agg()["sent"]
dataset["senders"] = senders.get_agg()["senders"]
dataset["senders_init"] = senders_init.get_agg()["senders_init"]
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
threads = Threads(filters.startdate, filters.enddate, dbcon.identities_db)
top_longest_threads = threads.topLongestThread(10)
top_longest_threads = serialize_threads(top_longest_threads, False, threads)
createJSON(top_longest_threads, "./release/mls_top_longest_threads.json")
createCSV(top_longest_threads, "./release/mls_top_longest_threads.csv")
top_crowded_threads = threads.topCrowdedThread(10)
top_crowded_threads = serialize_threads(top_crowded_threads, True, threads)
createJSON(top_crowded_threads, "./release/mls_top_crowded_threads.json")
createCSV(top_crowded_threads, "./release/mls_top_crowded_threads.csv")
return dataset
def test_get_filter_items(self):
opts = read_options()
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
for filter_ in Report.get_filters():
items = ds.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
if ds.get_name() in ["scr","pullpo"] :
items = [item.replace("/","_") for item in items]
elif ds.get_name() == "mls":
items = [item.replace("/","_").replace("<","__").replace(">","___")
for item in items]
fn = ds.get_name()+"-"+filter_.get_name_plural()+".json"
createJSON(items, opts.destdir+"/"+ fn)
test_json = os.path.join("json",fn)
new_json = opts.destdir+"/"+ fn
if ds.get_name() not in ["scr","pullpo"] :
# scr, pullpo repos format is more complex and
# is checked already in test_get_agg_evol_filters_data
self.assertTrue(self.compareJSON(test_json, new_json))
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items:
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate,
identities_db, filter_item,
npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir,
filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir, filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
if filter_name in ["people2","company","repository","country","domain","project"] :
filter_all = Filter(filter_name, None)
agg_all = SCR.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(SCR()))
createJSON(agg_all, fn)
SCR.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(SCR()))
createJSON(evol_all, fn)
SCR.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
SCR._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
SCR._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def result(self, data_source, destdir = None):
from vizgrimoire.SCR import SCR
if data_source != SCR or destdir is None: return
period = self.filters.period
startdate = self.filters.startdate
enddate = self.filters.enddate
code_contrib = {}
code_contrib["submitters"] = self.GetNewSubmitters()
code_contrib["mergers"] = self.GetNewMergers()
code_contrib["abandoners"] = self.GetNewAbandoners()
createJSON(code_contrib, destdir+"/scr-code-contrib-new.json")
code_contrib = {}
code_contrib["submitters"] = self.GetGoneSubmitters()
code_contrib["mergers"] = self.GetGoneMergers()
code_contrib["abandoners"] = self.GetGoneAbandoners()
createJSON(code_contrib, destdir+"/scr-code-contrib-gone.json")
data = self.GetNewSubmittersActivity()
evol = {}
evol['people'] = {}
for uuid in data['uuid']:
pdata = self.db.GetPeopleEvolSubmissionsSCR(uuid, period, startdate, enddate)
pdata = completePeriodIds(pdata, period, startdate, enddate)
evol['people'][uuid] = {"submissions":pdata['submissions']}
# Just to have the time series data
evol = dict(evol.items() + pdata.items())
if 'changes' in evol:
del evol['changes'] # closed (metrics) is included in people
createJSON(evol, destdir+"/new-people-activity-scr-evolutionary.json")
data = self.GetGoneSubmittersActivity()
evol = {}
evol['people'] = {}
for uuid in data['uuid']:
pdata = self.db.GetPeopleEvolSubmissionsSCR(uuid, period, startdate, enddate)
pdata = completePeriodIds(pdata, period, startdate, enddate)
evol['people'][uuid] = {"submissions":pdata['submissions']}
# Just to have the time series data
evol = dict(evol.items() + pdata.items())
if 'changes' in evol:
del evol['changes'] # closed (metrics) is included in people
createJSON(evol, destdir+"/gone-people-activity-scr-evolutionary.json")
# data = GetPeopleLeaving()
# createJSON(data, destdir+"/leaving-people-scr.json")
evol = {}
data = completePeriodIds(self.db.GetPeopleIntake(0,1), period, startdate, enddate)
evol[period] = data[period]
evol['id'] = data['id']
evol['date'] = data['date']
evol['num_people_1'] = data['people']
evol['num_people_1_5'] = completePeriodIds(self.db.GetPeopleIntake(1,5),period, startdate, enddate)['people']
evol['num_people_5_10'] = completePeriodIds(self.db.GetPeopleIntake(5,10), period, startdate, enddate)['people']
createJSON(evol, destdir+"/scr-people-intake-evolutionary.json")
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = cls.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
createJSON(evol_data, fn)
agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(agg, fn)
if filter_name in ["domain", "company", "repository"]:
items_list['name'].append(item.replace('/', '_'))
items_list['closed_365'].append(agg['closed_365'])
items_list['closers_365'].append(agg['closers_365'])
if filter_name in ["company","domain","repository"]:
top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(cls()))
createJSON(top, fn)
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = ITS
#summary = cls.get_filter_summary(
# filter_, period, startdate, enddate,
# identities_db, 10
# )
#createJSON (summary,
# destdir + "/" + filter_.get_summary_filename(cls))
# Perform ages study, if it is specified in Report
cls.ages_study_com (items, period, startdate, enddate, destdir)
def create_agg_report(period,
startdate,
enddate,
destdir,
i_db,
filter_=None):
data = IRC.get_agg_data(period, startdate, enddate, i_db, filter_)
filename = IRC().get_agg_filename()
createJSON(data, os.path.join(destdir, filename))
def result(self):
data = {}
data['resolvers.'] = self._query(0)
data['resolvers.last month'] = self._query(30)
data['resolvers.last year'] = self._query(365)
# TODO: Hardcoded creation of file
createJSON(data, "../../../../json/its-changers.json")
return data
def result(self, data_source = None, destdir = None):
fields = Set([])
tables = Set([])
filters = Set([])
fields.add("tr.url as project_name")
fields.add("pro.name as author_name")
fields.add("org.name as organization")
fields.add("i.issue as gerrit_issue_id")
fields.add("i.summary as summary")
fields.add("i.submitted_on as first_upload")
fields.add("t.last_upload as last_upload")
tables.add("issues i")
tables.add("trackers tr")
tables.add("people_uidentities puid")
tables.add(self.db.identities_db + ".enrollments enr")
tables.add(self.db.identities_db + ".organizations org")
tables.add(self.db.identities_db + ".profiles pro")
tables.add("(select issue_id, max(changed_on) as last_upload from changes where field='status' and new_value='UPLOADED' group by issue_id) t")
filters.add("t.issue_id = i.id")
filters.add("i.id not in (select distinct(issue_id) from changes where field='Code-Review')")
filters.add("i.status<>'Abandoned'")
filters.add("i.status<>'Merged'")
filters.add("tr.id=i.tracker_id")
filters.add("i.submitted_by=puid.people_id")
filters.add("puid.uuid = enr.uuid")
filters.add("i.submitted_on >= enr.start")
filters.add("i.submitted_on < enr.end")
filters.add("enr.organization_id = org.id")
filters.add("puid.uuid = pro.uuid")
filters.add("i.summary not like '%WIP%'")
query = " select " + self.db._get_fields_query(fields)
query = query + " from " + self.db._get_tables_query(tables)
query = query + " where " + self.db._get_filters_query(filters)
query = query + " order by org.name, t.last_upload"
data = self.db.ExecuteQuery(query)
# TODO: Hardcoded creation of file
createJSON(data, destdir + "/scr-oldest_changesets_by_affiliation.json")
#Filtering the data to have only 10 entries per organization at most
data_df = pandas.DataFrame(data, columns=["gerrit_issue_id", "project_name", "organization", "last_upload", "first_upload", "summary", "author_name"])
organizations = pandas.unique(data_df.organization)
dataframes = []
for organization in organizations:
dataframes.append(data_df[data_df["organization"]==organization][1:10])
filter_orgs = pandas.concat(dataframes)
filter_orgs = filter_orgs.to_dict(orient="list")
createJSON(filter_orgs, destdir + "/scr-oldest_changesets_by_affiliation.json")
return filter_orgs
def create_evolutionary_report(period,
startdate,
enddate,
destdir,
i_db,
type_analysis=None):
data = EventsDS.get_evolutionary_data(period, startdate, enddate, i_db,
type_analysis)
filename = EventsDS().get_evolutionary_filename()
createJSON(data, os.path.join(destdir, filename))
def create_evolutionary_report(period,
startdate,
enddate,
destdir,
i_db,
filter_=None):
data = DownloadsDS.get_evolutionary_data(period, startdate, enddate,
i_db, filter_)
filename = DownloadsDS().get_evolutionary_filename()
createJSON(data, os.path.join(destdir, filename))
def create_evolutionary_report(period,
startdate,
enddate,
destdir,
identities_db,
filter_=None):
data = IRC.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_)
filename = IRC().get_evolutionary_filename()
createJSON(data, os.path.join(destdir, filename))
def create_agg_report(period,
startdate,
enddate,
destdir,
i_db,
type_analysis=None):
data = ReleasesDS.get_agg_data(period, startdate, enddate, i_db,
type_analysis)
filename = ReleasesDS().get_agg_filename()
createJSON(data, os.path.join(destdir, filename))
def _compare_data(self, data, json_file):
# Create a temporary JSON file with data
from tempfile import NamedTemporaryFile
data_file = NamedTemporaryFile()
data_file_name = data_file.name
data_file.close()
createJSON(data, data_file_name, check=False, skip_fields = [])
check = self.compareJSON(json_file, data_file_name)
if check: os.remove(data_file_name)
return check
def create_filter_report_all(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
# New API for getting all metrics with one query
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
# This report is created per item, not using GROUP BY yet
SCM.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
# Filters metrics computed using GROUP BY queries
if filter_name in [
"people2", "company", "country", "repository", "domain",
"project", "company" + MetricFilters.DELIMITER + "country",
"company" + MetricFilters.DELIMITER + "project"
]:
filter_all = Filter(filter_name, None)
agg_all = SCM.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(SCM()))
createJSON(agg_all, fn)
SCM.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = SCM.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_evolutionary_filename_all(SCM()))
createJSON(evol_all, fn)
SCM.convert_all_to_single(evol_all, filter_, destdir, True, period)
# Studies report for filters
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com(agg_all['name'], period, startdate, enddate,
destdir)
# check is only done fpr basic filters. Composed should work if basic does.
if check and not "," in filter_name:
SCM._check_report_all_data(evol_all, filter_, startdate,
enddate, identities_db, True,
period)
SCM._check_report_all_data(agg_all, filter_, startdate,
enddate, identities_db, False,
period)
else:
logging.error(filter_name +
" does not support yet group by items sql queries")
def mls_report(dbcon, filters):
emails_sent = mls.EmailsSent(dbcon, filters)
createJSON(emails_sent.get_agg(), "./release/mls_emailsent.json")
print(emails_sent.get_agg())
emails_senders = mls.EmailsSenders(dbcon, filters)
createJSON(emails_senders.get_agg(), "./release/mls_emailsenders.json")
print(emails_senders.get_agg())
top = {}
from vizgrimoire.MLS import MLS
msenders = mls.EmailsSenders(dbcon, filters)
top["EmailSenders"] = msenders.get_list(filters, 90)
createJSON(top, "./release/mls_top_email_senders.json")
createCSV(top["EmailSenders"], "./release/mls_top_email_senders.csv",
['id'])
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
top_threads = {}
top_threads['threads'] = MLS.getLongestThreads(filters.startdate,
filters.enddate,
dbcon.identities_db,
str(filters.npeople))
createJSON(top_threads, "./release/mls_top_longest_threads.json")
createCSV(top_threads["threads"], "./release/mls_top_longest_threads.csv",
['initiator_id', 'message_id'])
main_topics = Threads(filters.startdate, filters.enddate,
dbcon.identities_db)
top_crowded = main_topics.topCrowdedThread(int(filters.npeople))
l_threads = {}
l_threads['message_id'] = []
l_threads['people'] = []
l_threads['subject'] = []
l_threads['date'] = []
l_threads['initiator_name'] = []
l_threads['initiator_id'] = []
l_threads['url'] = []
for email_people in top_crowded:
email = email_people[0]
l_threads['message_id'].append(email.message_id)
l_threads['people'].append(email_people[1])
l_threads['subject'].append(email.subject)
l_threads['date'].append(email.date.strftime("%Y-%m-%d"))
l_threads['initiator_name'].append(email.initiator_name)
l_threads['initiator_id'].append(email.initiator_id)
l_threads['url'].append(email.url)
createJSON(l_threads, "./release/mls_top_crowded_threads.json")
createCSV(l_threads, "./release/mls_top_crowded_threads.csv",
['initiator_id', 'message_id'])
def create_people_identifiers(startdate, enddate, destdir, npeople,
identities_db):
from vizgrimoire.GrimoireUtils import check_array_values
logging.info("Generating people identifiers")
people = get_top_report(startdate,
enddate,
npeople,
identities_db,
only_people=True)
people_ids = [] # upeople_ids which need identifiers
people_data = {} # identifiers for upeople_ids
ds_scm = Report.get_data_source("scm")
if ds_scm is None:
# Without SCM (identities) data source can not continue
return
for ds in Report.get_data_sources():
periods = [".", ".last year", ".last month"]
top_names = ds.get_top_metrics()
for period in periods:
for top_name in top_names:
if top_name + period in people[ds.get_name()]:
if 'id' in people[ds.get_name()][top_name + period]:
people_ids += check_array_values(
people[ds.get_name()][top_name + period])['id']
people_ids = list(set(people_ids))
from vizgrimoire.SCM import GetPeopleListSCM
import vizgrimoire.People as People
from vizgrimoire.GrimoireSQL import SetDBChannel
# TODO: Identities db is the same than SCM
Report.connect_ds(ds_scm)
for upeople_id in people_ids:
people_data[upeople_id] = People.GetPersonIdentifiers(
identities_db, upeople_id)
all_top_min_ds = get_top_people(startdate, enddate, identities_db)
db = automator['generic']['db_cvsanaly']
SetDBChannel(database=db, user=opts.dbuser, password=opts.dbpassword)
for upeople_id in all_top_min_ds:
people_data[upeople_id] = People.GetPersonIdentifiers(
identities_db, upeople_id)
createJSON(people_data, destdir + "/people.json")
return people_ids
def scm_report(dbcon, filters):
# Per release aggregated information
project_name = filters.type_analysis[1]
project_name = project_name.replace(" ", "")
commits = scm.Commits(dbcon, filters)
createJSON(commits.get_agg(), "./release/scm_commits_"+project_name+".json")
authors = scm.Authors(dbcon, filters)
createJSON(authors.get_agg(), "./release/scm_authors_"+project_name+".json")
dataset = {}
dataset["commits"] = commits.get_agg()["commits"]
dataset["authors"] = authors.get_agg()["authors"]
# tops authors activity
top_authors = authors.get_list()
if not isinstance(top_authors["commits"], list):
top_authors["commits"] = [top_authors["commits"]]
top_authors["id"] = [top_authors["id"]]
top_authors["authors"] = [top_authors["authors"]]
createJSON(top_authors, "./release/scm_top_authors_project_"+project_name+".json")
createCSV(top_authors, "./release/scm_top_authors_project_"+project_name+".csv", ["id"])
# top companies activity
companies = scm.Companies(dbcon, filters)
top_companies = companies.get_list(filters)
if not isinstance(top_companies["company_commits"], list):
top_companies["company_commits"] = [top_companies["company_commits"]]
top_companies["companies"] = [top_companies["name"]]
createJSON(top_companies, "./release/scm_top_companies_project_"+project_name+".json")
createCSV(top_companies, "./release/scm_top_companies_project_"+project_name+".csv")
return dataset
def mls_report(dbcon, filters):
# Per release MLS information
emails = mls.EmailsSent(dbcon, filters)
createJSON(emails.get_agg(), "./release/mls_emailssent.json")
senders = mls.EmailsSenders(dbcon, filters)
createJSON(senders.get_agg(), "./release/mls_emailssenders.json")
senders_init = mls.SendersInit(dbcon, filters)
createJSON(senders_init.get_agg(), "./release/mls_sendersinit.json")
dataset = {}
dataset["sent"] = emails.get_agg()["sent"]
dataset["senders"] = senders.get_agg()["senders"]
dataset["senders_init"] = senders_init.get_agg()["senders_init"]
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
threads = Threads(filters.startdate, filters.enddate, dbcon.identities_db)
top_longest_threads = threads.topLongestThread(10)
top_longest_threads = serialize_threads(top_longest_threads, False,
threads)
createJSON(top_longest_threads, "./release/mls_top_longest_threads.json")
createCSV(top_longest_threads, "./release/mls_top_longest_threads.csv")
#top_crowded_threads = threads.topCrowdedThread(10)
#top_crowded_threads = serialize_threads(top_crowded_threads, True, threads)
#createJSON(top_crowded_threads, "./release/mls_top_crowded_threads.json")
#createCSV(top_crowded_threads, "./release/mls_top_crowded_threads.csv")
return dataset
def scm_report(dbcon, filters):
# Per release aggregated information
project_name = "general"
commits = scm.Commits(dbcon, filters)
createJSON(commits.get_agg(), "./release/scm_commits_"+project_name+".json")
authors = scm.Authors(dbcon, filters)
createJSON(authors.get_agg(), "./release/scm_authors_"+project_name+".json")
dataset = {}
dataset["commits"] = commits.get_agg()["commits"]
dataset["authors"] = authors.get_agg()["authors"]
# tops authors activity
top_authors = authors.get_list()
if not isinstance(top_authors["commits"], list):
top_authors["commits"] = [top_authors["commits"]]
top_authors["id"] = [top_authors["id"]]
top_authors["authors"] = [top_authors["authors"]]
createJSON(top_authors, "./release/scm_top_authors_project_"+project_name+".json")
createCSV(top_authors, "./release/scm_top_authors_project_"+project_name+".csv", ["id"])
# top companies activity
companies = scm.Companies(dbcon, filters)
top_companies = companies.get_list(filters)
if not isinstance(top_companies["company_commits"], list):
top_companies["company_commits"] = [top_companies["company_commits"]]
top_companies["companies"] = [top_companies["name"]]
createJSON(top_companies, "./release/scm_top_companies_project_"+project_name+".json")
createCSV(top_companies, "./release/scm_top_companies_project_"+project_name+".csv")
return dataset
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
filter_name = filter_.get_name()
if filter_name == "people2" or filter_name == "company":
filter_all = Filter(filter_name, None)
agg_all = Mediawiki.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(Mediawiki()))
createJSON(agg_all, fn)
evol_all = Mediawiki.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(Mediawiki()))
createJSON(evol_all, fn)
else:
logging.error(Mediawiki.get_name()+ " " + filter_name +" does not support yet group by items sql queries")
def irc_report(dbcon, filters, output_dir):
# irc basic report
dataset = {}
sent = irc.Sent(dbcon, filters)
dataset["irc_sent"] = sent.get_agg()["sent"]
senders = irc.Senders(dbcon, filters)
dataset["irc_senders"] = senders.get_agg()["senders"]
top_senders = senders.get_list()
createJSON(top_senders, output_dir + "/irc_top_senders.json")
createCSV(top_senders, output_dir + "/irc_top_senders.csv")
return dataset
def irc_report(dbcon, filters, output_dir):
# irc basic report
dataset = {}
sent = irc.Sent(dbcon, filters)
dataset["irc_sent"] = sent.get_agg()["sent"]
senders = irc.Senders(dbcon, filters)
dataset["irc_senders"] = senders.get_agg()["senders"]
top_senders = senders.get_list()
createJSON(top_senders, output_dir + "/irc_top_senders.json")
createCSV(top_senders, output_dir + "/irc_top_senders.csv")
return dataset
def scm_report(dbcon, filters):
commits = scm.Commits(dbcon, filters)
createJSON(commits.get_agg(), "./release/scm_commits.json")
print(commits.get_agg())
authors = scm.Authors(dbcon, filters)
createJSON(authors.get_agg(), "./release/scm_authors.json")
print(authors.get_agg())
from vizgrimoire.analysis.contributors_new_gone import ContributorsNewGoneSCM
from vizgrimoire.SCM import SCM
newcommers_leavers = ContributorsNewGoneSCM(dbcon, filters)
newcommers_leavers_dict = newcommers_leavers.result(SCM, "./release/")
# createJSON(newcommers_leavers_dict, "./release/scm_newcommers_leavers.json")
createCSV(newcommers_leavers_dict['people_new']["authors"],
"./release/scm_top_authors_new.csv",
['revtime', 'author_id', 'upeople_id'])
createCSV(newcommers_leavers_dict['people_gone']["authors"],
"./release/scm_top_authors_gone.csv",
['revtime', 'author_id', 'upeople_id'])
top_authors = {}
top_authors["authors"] = scm.Authors(dbcon, filters).get_list(filters, 0)
createJSON(top_authors, "./release/scm_top_authors.json")
createCSV(top_authors["authors"], "./release/scm_top_authors.csv", ['id'])
top_repos = scm.Repositories(dbcon, filters).get_list()
createJSON(top_repos, "./release/scm_top_repositories.json")
createCSV(top_repos, "./release/scm_top_repositories.csv")
from vizgrimoire.analysis.onion_transitions import OnionTransitions
scm_ds = SCM()
ot = OnionTransitions(scm_dbcon, filters)
ot_data = ot.result(scm_ds)
#= {"up_core":up_core, "up_reg":up_reg, "down_reg":down_reg, "down_occ":down_occ}
createCSV(ot_data["core"], "./release/scm_core-shortterm.csv")
createCSV(ot_data["up_reg"],
"./release/scm_promoted_to_regular-shortterm.csv")
createCSV(ot_data["up_core"],
"./release/scm_promoted_to_core-shortterm.csv")
createCSV(ot_data["down_occ"],
"./release/scm_relegated_to_occasional-shortterm.csv")
createCSV(ot_data["down_reg"],
"./release/scm_relegated_to_regular-shortterm.csv")
ot_data = ot.result(scm_ds, 365)
#= {"up_core":up_core, "up_reg":up_reg, "down_reg":down_reg, "down_occ":down_occ}
createCSV(ot_data["core"], "./release/scm_core-longterm.csv")
createCSV(ot_data["up_reg"],
"./release/scm_promoted_to_regular-longterm.csv")
createCSV(ot_data["up_core"],
"./release/scm_promoted_to_core-longterm.csv")
createCSV(ot_data["down_occ"],
"./release/scm_relegated_to_occasional-longterm.csv")
createCSV(ot_data["down_reg"],
"./release/scm_relegated_to_regular-longterm.csv")
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_" + item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name': [], 'events_365': [], 'rsvps_365': []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate,
enddate, identities_db,
filter_item)
fn = os.path.join(
destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def mls_report(dbcon, filters):
emails_sent = mls.EmailsSent(dbcon, filters)
createJSON(emails_sent.get_agg(), "./release/mls_emailsent.json")
print(emails_sent.get_agg())
emails_senders = mls.EmailsSenders(dbcon, filters)
createJSON(emails_senders.get_agg(), "./release/mls_emailsenders.json")
print(emails_senders.get_agg())
top = {}
from vizgrimoire.MLS import MLS
msenders = mls.EmailsSenders(dbcon, filters)
top["EmailSenders"] = msenders.get_list(filters, 90)
createJSON(top, "./release/mls_top_email_senders.json")
createCSV(top["EmailSenders"], "./release/mls_top_email_senders.csv", ['id'])
from vizgrimoire.analysis.threads import Threads
SetDBChannel(dbcon.user, dbcon.password, dbcon.database)
top_threads = {}
top_threads['threads'] = MLS.getLongestThreads(filters.startdate, filters.enddate, dbcon.identities_db, str(filters.npeople))
createJSON(top_threads, "./release/mls_top_longest_threads.json")
createCSV(top_threads["threads"], "./release/mls_top_longest_threads.csv", ['initiator_id','message_id'])
main_topics= Threads(filters.startdate, filters.enddate, dbcon.identities_db)
top_crowded = main_topics.topCrowdedThread(filters.npeople)
l_threads = {}
l_threads['message_id'] = []
l_threads['people'] = []
l_threads['subject'] = []
l_threads['date'] = []
l_threads['initiator_name'] = []
l_threads['initiator_id'] = []
l_threads['url'] = []
for email_people in top_crowded:
email = email_people[0]
l_threads['message_id'].append(email.message_id)
l_threads['people'].append(email_people[1])
l_threads['subject'].append(email.subject)
l_threads['date'].append(email.date.strftime("%Y-%m-%d"))
l_threads['initiator_name'].append(email.initiator_name)
l_threads['initiator_id'].append(email.initiator_id)
l_threads['url'].append(email.url)
createJSON(l_threads, "./release/mls_top_crowded_threads.json")
createCSV(l_threads, "./release/mls_top_crowded_threads.csv", ['initiator_id','message_id'])
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
# New API for getting all metrics with one query
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
# This report is created per item, not using GROUP BY yet
SCM.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
# Filters metrics computed using GROUP BY queries
if filter_name in ["people2","company", "country","repository","domain","project",
"company"+MetricFilters.DELIMITER+"country",
"company"+MetricFilters.DELIMITER+"project"]:
filter_all = Filter(filter_name, None)
agg_all = SCM.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(SCM()))
createJSON(agg_all, fn)
SCM.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = SCM.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(SCM()))
createJSON(evol_all, fn)
SCM.convert_all_to_single(evol_all, filter_, destdir, True, period)
# Studies report for filters
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com (agg_all['name'], period, startdate, enddate, destdir)
# check is only done fpr basic filters. Composed should work if basic does.
if check and not "," in filter_name:
SCM._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
SCM._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def create_people_identifiers(startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.GrimoireUtils import check_array_values
logging.info("Generating people identifiers")
people = get_top_report(startdate, enddate, npeople, identities_db);
people_ids = [] # upeople_ids which need identifiers
people_data = {} # identifiers for upeople_ids
ds_scm = Report.get_data_source("scm")
if ds_scm is None:
# Without SCM (identities) data source can not continue
return
for ds in Report.get_data_sources():
periods = [".",".last year",".last month"]
top_names = ds.get_top_metrics();
for period in periods:
for top_name in top_names:
if top_name+period in people[ds.get_name()]:
if 'id' in people[ds.get_name()][top_name+period]:
people_ids += check_array_values(people[ds.get_name()][top_name+period])['id']
people_ids = list(set(people_ids))
from vizgrimoire.SCM import GetPeopleListSCM
import vizgrimoire.People as People
from vizgrimoire.GrimoireSQL import SetDBChannel
# TODO: Identities db is the same than SCM
Report.connect_ds(ds_scm)
for upeople_id in people_ids:
people_data[upeople_id] = People.GetPersonIdentifiers(identities_db, upeople_id)
all_top_min_ds = get_top_people(startdate, enddate, identities_db)
db = automator['generic']['db_cvsanaly']
SetDBChannel (database=db, user=opts.dbuser, password=opts.dbpassword)
for upeople_id in all_top_min_ds:
people_data[upeople_id] = People.GetPersonIdentifiers(identities_db, upeople_id)
createJSON(people_data, destdir+"/people.json")
return people_ids
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items :
item = item.replace("'", "\\'")
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False)
createJSON(top_senders, destdir+"/"+filter_item.get_top_filename(MLS()))
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items:
# item_name = "'"+ item+ "'"
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(IRC()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = QAForums.get_filter_items(filter_, startdate, enddate, identities_db)
if items == None:
return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(QAForums()))
createJSON(file_items, fn)
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = QAForums.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(QAForums()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = QAForums.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(QAForums()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items :
# item_name = "'"+ item+ "'"
logging.info (item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(IRC()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_json(dbcon, filters):
ITS.set_backend("jira")
ITS._get_backend().closed_condition = " i.status = 'Closed' "
pullrequests = PullRequests(dbcon, filters)
submitters = Submitters(dbcon, filters)
abandoned = Abandoned(dbcon, filters)
merged = Merged(dbcon, filters)
trackers = Trackers(dbcon, filters)
#timeseries data
data = dict(pullrequests.get_ts().items() +
submitters.get_ts().items() +
abandoned.get_ts().items() +
merged.get_ts().items() +
trackers.get_ts().items())
if filters.type_analysis == []:
createJSON(data, "scr-evolutionary.json")
else:
tracker = filters.type_analysis[1]
tracker = tracker.replace("'", "")
name = tracker.split("/")[-1:][0]
createJSON(data, name + "-scr-evolutionary.json")
#aggregated data
data = dict(pullrequests.get_agg().items() +
submitters.get_agg().items() +
abandoned.get_agg().items() +
merged.get_agg().items() +
trackers.get_agg().items())
enddate = filters.enddate
for i in [7, 365, 0]:
data = dict(data.items() +
pullrequests.get_trends(enddate, i).items() +
submitters.get_trends(enddate, i).items() +
abandoned.get_trends(enddate, i).items() +
merged.get_trends(enddate, i).items() +
trackers.get_trends(enddate, i).items())
if filters.type_analysis == []:
createJSON(data, "scr-static.json")
else:
tracker = filters.type_analysis[1]
tracker = tracker.replace("'", "")
name = tracker.split("/")[-1:][0]
createJSON(data, name + "-scr-static.json")
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug pull_requests
filter_name = filter_.get_name()
if filter_name == "people2" or filter_name == "company+country":
filter_all = Filter(filter_name, None)
agg_all = Pullpo.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(Pullpo()))
createJSON(agg_all, fn)
evol_all = Pullpo.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(Pullpo()))
createJSON(evol_all, fn)
if check:
Pullpo._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
Pullpo._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def its_report(dbcon, filters):
# Per release its information
from vizgrimoire.ITS import ITS
ITS.set_backend("launchpad")
project_name = filters.type_analysis[1]
project_name = project_name.replace(" ", "")
if project_name == 'Documentation':
ITS._get_backend().closed_condition = "(new_value='Fix Committed' or new_value='Fix Released')"
else:
ITS.closed_condition = "(new_value='Fix Committed')"
opened = its.Opened(dbcon, filters)
createJSON(opened.get_agg(), "./release/its_opened_"+project_name+".json")
closed = its.Closed(dbcon, filters)
createJSON(closed.get_agg(), "./release/its_closed_"+project_name+".json")
dataset = {}
dataset["opened"] = opened.get_agg()["opened"]
dataset["closed"] = closed.get_agg()["closed"]
return dataset
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name' : [], 'events_365' : [], 'rsvps_365' : []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def result(self, data_source = None, destdir = None):
fields = Set([])
tables = Set([])
filters = Set([])
fields.add("tr.url as project_name")
fields.add("p.name as author_name")
fields.add("i.issue as gerrit_issue_id")
fields.add("i.summary as summary")
fields.add("i.submitted_on as first_upload")
fields.add("t.last_upload as last_upload")
tables.add("issues i")
tables.add("trackers tr")
tables.add("people p")
tables.add("(select issue_id, max(changed_on) as last_upload from changes where field='status' and new_value='UPLOADED' group by issue_id) t")
filters.add("t.issue_id = i.id")
filters.add("i.id not in (select distinct(issue_id) from changes where field='Code-Review')")
filters.add("i.status<>'Abandoned'")
filters.add("i.status<>'Merged'")
filters.add("tr.id=i.tracker_id")
filters.add("i.submitted_by=p.id")
filters.add("i.summary not like '%WIP%'")
query = " select " + self.db._get_fields_query(fields)
query = query + " from " + self.db._get_tables_query(tables)
query = query + " where " + self.db._get_filters_query(filters)
query = query + " order by last_upload limit 100"
data = self.db.ExecuteQuery(query)
# TODO: Hardcoded creation of file
createJSON(data, destdir + "/scr-oldest_changesets.json")
return data
def scm_report(dbcon, filters):
commits = scm.Commits(dbcon, filters)
createJSON(commits.get_agg(), "./release/scm_commits.json")
print(commits.get_agg())
authors = scm.Authors(dbcon, filters)
createJSON(authors.get_agg(), "./release/scm_authors.json")
print(authors.get_agg())
from vizgrimoire.analysis.contributors_new_gone import ContributorsNewGoneSCM
from vizgrimoire.SCM import SCM
newcommers_leavers = ContributorsNewGoneSCM(dbcon, filters)
newcommers_leavers_dict = newcommers_leavers.result(SCM, "./release/")
# createJSON(newcommers_leavers_dict, "./release/scm_newcommers_leavers.json")
createCSV(newcommers_leavers_dict['people_new']["authors"], "./release/scm_top_authors_new.csv", ['revtime','author_id','upeople_id'])
createCSV(newcommers_leavers_dict['people_gone']["authors"], "./release/scm_top_authors_gone.csv", ['revtime','author_id','upeople_id'])
top_authors = {}
top_authors["authors"] = scm.Authors(dbcon, filters).get_list(filters, 0)
createJSON(top_authors, "./release/scm_top_authors.json")
createCSV(top_authors["authors"], "./release/scm_top_authors.csv",['id'])
top_repos = scm.Repositories(dbcon, filters).get_list()
createJSON(top_repos, "./release/scm_top_repositories.json")
createCSV(top_repos, "./release/scm_top_repositories.csv")
from vizgrimoire.analysis.onion_transitions import OnionTransitions
scm_ds = SCM()
ot = OnionTransitions(scm_dbcon, filters)
ot_data = ot.result(scm_ds)
#= {"up_core":up_core, "up_reg":up_reg, "down_reg":down_reg, "down_occ":down_occ}
createCSV(ot_data["core"], "./release/scm_core-shortterm.csv")
createCSV(ot_data["up_reg"], "./release/scm_promoted_to_regular-shortterm.csv")
createCSV(ot_data["up_core"], "./release/scm_promoted_to_core-shortterm.csv")
createCSV(ot_data["down_occ"], "./release/scm_relegated_to_occasional-shortterm.csv")
createCSV(ot_data["down_reg"], "./release/scm_relegated_to_regular-shortterm.csv")
ot_data = ot.result(scm_ds, 365)
#= {"up_core":up_core, "up_reg":up_reg, "down_reg":down_reg, "down_occ":down_occ}
createCSV(ot_data["core"], "./release/scm_core-longterm.csv")
createCSV(ot_data["up_reg"], "./release/scm_promoted_to_regular-longterm.csv")
createCSV(ot_data["up_core"], "./release/scm_promoted_to_core-longterm.csv")
createCSV(ot_data["down_occ"], "./release/scm_relegated_to_occasional-longterm.csv")
createCSV(ot_data["down_reg"], "./release/scm_relegated_to_regular-longterm.csv")
def qaforums_report(dbcon, filters, output_dir):
# basic metrics for qaforums
dataset = {}
questions = qa.Questions(dbcon, filters)
dataset["qa_questions"] = questions.get_agg()["qsent"]
answers = qa.Answers(dbcon, filters)
dataset["qa_answers"] = answers.get_agg()["asent"]
comments = qa.Comments(dbcon, filters)
dataset["qa_comments"] = comments.get_agg()["csent"]
q_senders = qa.QuestionSenders(dbcon, filters)
dataset["qa_qsenders"] = q_senders.get_agg()["qsenders"]
import vizgrimoire.analysis.top_questions_qaforums as top
tops = top.TopQuestions(dbcon, filters)
commented = tops.top_commented()
commented["qid"] = commented.pop("question_identifier")
# Taking the last part of the URL
commented["site"] = parse_urls(commented.pop("url"))
createJSON(commented, output_dir + "/qa_top_questions_commented.json")
createCSV(commented, output_dir + "/qa_top_questions_commented.csv")
visited = tops.top_visited()
visited["qid"] = visited.pop("question_identifier")
visited["site"] = parse_urls(visited.pop("url"))
createJSON(visited, output_dir + "/qa_top_questions_visited.json")
createCSV(visited, output_dir + "/qa_top_questions_visited.csv")
crowded = tops.top_crowded()
crowded["qid"] = crowded.pop("question_identifier")
crowded["site"] = parse_urls(crowded.pop("url"))
createJSON(crowded, output_dir + "/qa_top_questions_crowded.json")
createCSV(crowded, output_dir + "./qa_top_questions_crowded.csv")
filters.npeople = 15
createJSON(tops.top_tags(), output_dir + "/qa_top_tags.json")
createCSV(tops.top_tags(), output_dir + "/qa_top_tags.csv")
return dataset
def irc_report(dbcon, filters):
# per release information for IRC
pass
sent = irc.Sent(dbcon, filters)
createJSON(sent.get_agg(), "./release/irc_sent.json")
senders = irc.Senders(dbcon, filters)
createJSON(senders.get_agg(), "./release/irc_senders.json")
dataset = {}
dataset["sent"] = sent.get_agg()["sent"]
dataset["senders"] = senders.get_agg()["senders"]
top_senders = senders.get_list()
createJSON(top_senders, "./release/irc_top_senders.json")
createCSV(top_senders, "./release/irc_top_senders.csv")
return dataset