def countriesData(period, startdate, enddate, idb, destdir):
# countries = dataFrame2Dict(vizr.GetCountriesSCRName(startdate, enddate, idb))
countries = SCR.GetCountriesSCRName(startdate, enddate, idb)
countries = countries['name']
countries_files = [country.replace('/', '_') for country in countries]
createJSON(countries_files, destdir+"/scr-countries.json")
# missing information from the rest of type of reviews, patches and
# number of patches waiting for reviewer and submitter
for country in countries:
country_file = country.replace("/","_")
type_analysis = ['country', country]
# Evol
evol = {}
# data = vizr.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = SCR.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsMerged(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsAbandoned(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
# TODO: when empty abandoned does not appeat at all in R JSON
createJSON(evol, destdir+ "/"+country_file+"-scr-cou-evolutionary.json",False)
# Static
agg = {}
# data = vizr.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# agg = dict(agg.items() + dataFrame2Dict(data).items())
data = SCR.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsMerged(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsAbandoned(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
createJSON(agg, destdir+"/"+country_file+"-scr-cou-static.json")
def tsData(period, startdate, enddate, identities_db, destdir, granularity,
conf, backend):
closed_condition = backend.closed_condition
# data = vizr.EvolITSInfo(period, startdate, enddate, identities_db, closed_condition = closed_condition)
# evol = completePeriodIds(dataFrame2Dict(data))
data = ITS.EvolITSInfo(period, startdate, enddate, identities_db, [],
closed_condition)
evol = completePeriodIds(data)
if ('companies' in reports):
data = ITS.EvolIssuesCompanies(period, startdate, enddate,
identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('countries' in reports):
data = ITS.EvolIssuesCountries(period, startdate, enddate,
identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('repositories' in reports):
data = ITS.EvolIssuesRepositories(period, startdate, enddate,
identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('domains' in reports):
data = ITS.EvolIssuesDomains(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
data = ticketsStates(period, startdate, enddate, identities_db, backend)
evol = dict(evol.items() + data.items())
createJSON(evol, destdir + "/its-evolutionary.json")
def GetSentSummaryCompanies (period, startdate, enddate, identities_db, num_companies):
count = 1
first_companies = {}
companies = companiesNames(identities_db, startdate, enddate, ["-Bot", "-Individual", "-Unknown"])
for company in companies:
type_analysis = ["company", "'"+company+"'"]
sent = EvolEmailsSent(period, startdate, enddate, identities_db, type_analysis)
sent = completePeriodIds(sent)
# Rename field sent to company name
sent[company] = sent["sent"]
del sent['sent']
if (count <= num_companies):
#Case of companies with entity in the dataset
first_companies = dict(first_companies.items() + sent.items())
else :
#Case of companies that are aggregated in the field Others
if 'Others' not in first_companies:
first_companies['Others'] = sent[company]
else:
first_companies['Others'] = [a+b for a, b in zip(first_companies['Others'],sent[company])]
count = count + 1
first_companies = completePeriodIds(first_companies)
return(first_companies)
def tsData(period, startdate, enddate, identities_db, destdir, granularity,
conf, backend):
closed_condition = backend.closed_condition
# data = vizr.EvolITSInfo(period, startdate, enddate, identities_db, closed_condition = closed_condition)
# evol = completePeriodIds(dataFrame2Dict(data))
data = ITS.EvolITSInfo(period, startdate, enddate, identities_db, [], closed_condition)
evol = completePeriodIds(data)
if ('companies' in reports) :
data = ITS.EvolIssuesCompanies(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('countries' in reports) :
data = ITS.EvolIssuesCountries(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('repositories' in reports) :
data = ITS.EvolIssuesRepositories(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('domains' in reports) :
data = ITS.EvolIssuesDomains(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
data = ticketsStates(period, startdate, enddate, identities_db, backend)
evol = dict(evol.items() + data.items())
createJSON (evol, destdir+"/its-evolutionary.json")
def GetSentSummaryCompanies(period, startdate, enddate, identities_db,
num_companies):
count = 1
first_companies = {}
companies = companiesNames(identities_db, startdate, enddate,
["-Bot", "-Individual", "-Unknown"])
for company in companies:
type_analysis = ["company", "'" + company + "'"]
sent = EvolEmailsSent(period, startdate, enddate, identities_db,
type_analysis)
sent = completePeriodIds(sent)
# Rename field sent to company name
sent[company] = sent["sent"]
del sent['sent']
if (count <= num_companies):
#Case of companies with entity in the dataset
first_companies = dict(first_companies.items() + sent.items())
else:
#Case of companies that are aggregated in the field Others
if 'Others' not in first_companies:
first_companies['Others'] = sent[company]
else:
first_companies['Others'] = [
a + b
for a, b in zip(first_companies['Others'], sent[company])
]
count = count + 1
first_companies = completePeriodIds(first_companies)
return (first_companies)
def GetClosedSummaryCompanies (period, startdate, enddate, identities_db, closed_condition, num_companies):
count = 1
first_companies = {}
companies = GetCompaniesNameITS(startdate, enddate, identities_db, closed_condition, ["-Bot", "-Individual", "-Unknown"])
companies = companies['name']
for company in companies:
type_analysis = ["company", "'"+company+"'"]
closed = EvolIssuesClosed(period, startdate, enddate, identities_db, type_analysis, closed_condition)
closed = completePeriodIds(closed)
# Rename field closed to company name
closed[company] = closed["closed"]
del closed['closed']
if (count <= num_companies):
#Case of companies with entity in the dataset
first_companies = dict(first_companies.items() + closed.items())
else :
#Case of companies that are aggregated in the field Others
if 'Others' not in first_companies:
first_companies['Others'] = closed[company]
else:
first_companies['Others'] = [a+b for a, b in zip(first_companies['Others'],closed[company])]
count = count + 1
first_companies = completePeriodIds(first_companies)
return(first_companies)
def GetClosedSummaryCompanies(period, startdate, enddate, identities_db,
closed_condition, num_companies):
count = 1
first_companies = {}
companies = GetCompaniesNameITS(startdate, enddate, identities_db,
closed_condition,
["-Bot", "-Individual", "-Unknown"])
companies = companies['name']
for company in companies:
type_analysis = ["company", "'" + company + "'"]
closed = EvolIssuesClosed(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
closed = completePeriodIds(closed)
# Rename field closed to company name
closed[company] = closed["closed"]
del closed['closed']
if (count <= num_companies):
#Case of companies with entity in the dataset
first_companies = dict(first_companies.items() + closed.items())
else:
#Case of companies that are aggregated in the field Others
if 'Others' not in first_companies:
first_companies['Others'] = closed[company]
else:
first_companies['Others'] = [
a + b
for a, b in zip(first_companies['Others'], closed[company])
]
count = count + 1
first_companies = completePeriodIds(first_companies)
return (first_companies)
def companiesData(period, startdate, enddate, idb, destdir):
# companies = dataFrame2Dict(vizr.GetCompaniesSCRName(startdate, enddate, idb))
companies = SCR.GetCompaniesSCRName(startdate, enddate, idb)
companies = companies['name']
companies_files = [company.replace('/', '_') for company in companies]
createJSON(companies_files, destdir + "/scr-companies.json")
# missing information from the rest of type of reviews, patches and
# number of patches waiting for reviewer and submitter
for company in companies:
company_file = company.replace("/", "_")
type_analysis = ['company', company]
# Evol
evol = {}
# data = vizr.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = SCR.EvolReviewsSubmitted(period, startdate, enddate,
type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsMerged(period, startdate, enddate, type_analysis,
idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsAbandoned(period, startdate, enddate,
type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
if (period == "month"):
data = SCR.EvolTimeToReviewSCR(period, startdate, enddate, idb,
type_analysis)
data['review_time_days_avg'] = checkFloatArray(
data['review_time_days_avg'])
data['review_time_days_median'] = checkFloatArray(
data['review_time_days_median'])
evol = dict(evol.items() + completePeriodIds(data).items())
createJSON(evol,
destdir + "/" + company_file + "-scr-com-evolutionary.json")
# Static
agg = {}
# data = vizr.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# agg = dict(agg.items() + dataFrame2Dict(data).items())
data = SCR.StaticReviewsSubmitted(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsMerged(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsAbandoned(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticTimeToReviewSCR(startdate, enddate, idb,
type_analysis)
val = data['review_time_days_avg']
if (not val or val == 0): data['review_time_days_avg'] = 0
else: data['review_time_days_avg'] = float(val)
val = data['review_time_days_median']
if (not val or val == 0): data['review_time_days_median'] = 0
else: data['review_time_days_median'] = float(val)
agg = dict(agg.items() + data.items())
createJSON(agg, destdir + "/" + company_file + "-scr-com-static.json")
def reposData(period, startdate, enddate, idb, destdir, conf):
repos = SCR.GetReposSCRName(startdate, enddate)
repos = repos["name"]
# For repos aggregated data. Include metrics to sort in javascript.
repos_list = {"name":[],"review_time_days_median":[],"submitted":[]}
# missing information from the rest of type of reviews, patches and
# number of patches waiting for reviewer and submitter
for repo in repos:
repo_file = repo.replace("/","_")
logging.info(repo_file)
repos_list["name"].append(repo_file)
# logging.info("Repo: " + repo_file)
type_analysis = ['repository', repo]
evol = {}
# data = vizr.EvolReviewsSubmitted(period, startdate, enddate, type_analysis)
data = SCR.EvolReviewsSubmitted(period, startdate, enddate, type_analysis)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsMerged(period, startdate, enddate, type_analysis)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsAbandoned(period, startdate, enddate, type_analysis)
evol = dict(evol.items() + completePeriodIds(data).items())
# data = vizr.EvolReviewsPendingChanges(period, startdate, enddate, conf, type_analysis)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = SCR.EvolReviewsPending(period, startdate, enddate, conf, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
if (period == "month"):
data = SCR.EvolTimeToReviewSCR(period, startdate, enddate, idb, type_analysis)
data['review_time_days_avg'] = checkFloatArray(data['review_time_days_avg'])
data['review_time_days_median'] = checkFloatArray(data['review_time_days_median'])
evol = dict(evol.items() + completePeriodIds(data).items())
createJSON(evol, destdir+ "/"+repo_file+"-scr-rep-evolutionary.json")
# Static
agg = {}
data = SCR.StaticReviewsSubmitted(period, startdate, enddate, type_analysis)
repos_list["submitted"].append(data["submitted"])
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsMerged(period, startdate, enddate, type_analysis)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsAbandoned(period, startdate, enddate, type_analysis)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsPending(period, startdate, enddate, type_analysis)
agg = dict(agg.items() + data.items())
data = SCR.StaticTimeToReviewSCR(startdate, enddate, idb, type_analysis)
val = data['review_time_days_avg']
if (not val or val == 0): data['review_time_days_avg'] = 0
else: data['review_time_days_avg'] = float(val)
val = data['review_time_days_median']
if (not val or val == 0): data['review_time_days_median'] = 0
else: data['review_time_days_median'] = float(val)
agg = dict(agg.items() + data.items())
repos_list["review_time_days_median"].append(data['review_time_days_median'])
createJSON(agg, destdir + "/"+repo_file + "-scr-rep-static.json")
createJSON(repos_list, destdir+"/scr-repos.json")
def GetEvolDataMediaWiki (period, startdate, enddate, i_db, type_analysis):
# 1- Retrieving information
reviews = completePeriodIds(EvolReviewsMediaWiki(period, startdate, enddate, i_db, type_analysis))
authors = completePeriodIds(EvolAuthorsMediaWiki(period, startdate, enddate, i_db, type_analysis))
pages = completePeriodIds(EvolPagesMediaWiki(period, startdate, enddate, i_db, type_analysis))
# 2- Merging information
evol_data = dict(reviews.items()+ authors.items() + pages.items())
return (evol_data)
def GetEvolDataMediaWiki(period, startdate, enddate, i_db, type_analysis):
# 1- Retrieving information
reviews = completePeriodIds(
EvolReviewsMediaWiki(period, startdate, enddate, i_db, type_analysis))
authors = completePeriodIds(
EvolAuthorsMediaWiki(period, startdate, enddate, i_db, type_analysis))
pages = completePeriodIds(
EvolPagesMediaWiki(period, startdate, enddate, i_db, type_analysis))
# 2- Merging information
evol_data = dict(reviews.items() + authors.items() + pages.items())
return (evol_data)
def reposData(period, startdate, enddate, identities_db, destdir, conf,
closed_condition):
# repos = dataFrame2Dict(vizr.GetReposNameITS(startdate, enddate))
repos = ITS.GetReposNameITS(startdate, enddate)
repos = repos['name']
if not isinstance(repos, (list)):
repos = [repos]
createJSON(repos, destdir + "/its-repos.json", False)
else:
createJSON(repos, destdir + "/its-repos.json")
for repo in repos:
repo_name = "'" + repo + "'"
repo_file = repo.replace("/", "_")
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db,
['repository', repo_name], closed_condition)
evol = completePeriodIds(evol)
if (repo_file == "http:__tracker.ceph.com_projects_rados-java_"):
createJSON(
evol, destdir + "/" + repo_file + "-its-rep-evolutionary.json",
False)
else:
createJSON(
evol, destdir + "/" + repo_file + "-its-rep-evolutionary.json")
agg = ITS.AggITSInfo(period, startdate, enddate, identities_db,
['repository', repo_name], closed_condition)
createJSON(agg, destdir + "/" + repo_file + "-its-rep-static.json")
def peopleData(period, startdate, enddate, identities_db, destdir,
closed_condition, top_data):
top = top_data['closers.']["id"]
top += top_data['closers.last year']["id"]
top += top_data['closers.last month']["id"]
top += top_data['openers.']["id"]
top += top_data['openers.last year']["id"]
top += top_data['openers.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir + "/its-people.json", False)
for upeople_id in people:
evol = ITS.GetPeopleEvolITS(upeople_id, period, startdate, enddate,
closed_condition)
evol = completePeriodIds(evol)
createJSON(
evol,
destdir + "/people-" + str(upeople_id) + "-its-evolutionary.json")
data = ITS.GetPeopleStaticITS(upeople_id, startdate, enddate,
closed_condition)
createJSON(data,
destdir + "/people-" + str(upeople_id) + "-its-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir,
closed_condition, bots, npeople):
# domains = dataFrame2Dict(vizr.GetDomainsNameITS(startdate, enddate, identities_db, closed_condition, bots))
domains = ITS.GetDomainsNameITS(startdate, enddate, identities_db,
closed_condition, bots)
domains = domains['name']
createJSON(domains, destdir + "/its-domains.json")
for domain in domains:
domain_name = "'" + domain + "'"
print(domain_name)
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db,
['domain', domain_name], closed_condition)
evol = completePeriodIds(evol)
createJSON(evol, destdir + "/" + domain + "-its-dom-evolutionary.json")
agg = ITS.AggITSInfo(period, startdate, enddate, identities_db,
['domain', domain_name], closed_condition)
createJSON(agg, destdir + "/" + domain + "-its-dom-static.json")
top = ITS.GetDomainTopClosers(domain_name, startdate, enddate,
identities_db, bots, closed_condition,
npeople)
createJSON(top, destdir + "/" + domain + "-its-dom-top-closers.json",
False)
def EvolReviewsPendingChanges(period, startdate, enddate, config, type_analysis = [], identities_db=None):
data = EvolReviewsSubmitted(period, startdate, enddate, type_analysis, identities_db)
data = completePeriodIds(data)
data1 = EvolReviewsMergedChanges(period, startdate, enddate, type_analysis, identities_db)
data1 = completePeriodIds(data1)
data2 = EvolReviewsAbandonedChanges(period, startdate, enddate, type_analysis, identities_db)
data2 = completePeriodIds(data2)
evol = dict(data.items() + data1.items() + data2.items())
pending = {"pending":[]}
for i in range(0,len(evol['merged_changes'])):
pending_val = evol["submitted"][i] - evol["merged_changes"][i] - evol["abandoned_changes"][i]
pending["pending"].append(pending_val)
pending["month"] = evol["month"]
pending = completePeriodIds(pending)
return pending
def reposData(period, startdate, enddate, identities_db, destdir, conf, repofield, npeople):
repos = MLS.reposNames(rfield, startdate, enddate)
createJSON (repos, destdir+"/mls-lists.json")
repos = repos['mailing_list_url']
check = True
if not isinstance(repos, (list)):
repos = [repos]
check = False
repos_files = [repo.replace('/', '_').replace("<","__").replace(">","___")
for repo in repos]
createJSON(repos_files, destdir+"/mls-repos.json", check)
for repo in repos:
# Evol data
repo_name = "'"+repo+"'"
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, ["repository", repo_name])
data = completePeriodIds(data)
listname_file = repo.replace("/","_").replace("<","__").replace(">","___")
# TODO: Multilist approach. We will obsolete it in future
createJSON (data, destdir+"/mls-"+listname_file+"-rep-evolutionary.json")
# Multirepos filename
createJSON (data, destdir+"/"+listname_file+"-mls-rep-evolutionary.json")
top_senders = MLS.repoTopSenders (repo, identities_db, startdate, enddate, repofield, npeople)
createJSON(top_senders, destdir+ "/"+listname_file+"-mls-rep-top-senders.json", False)
# Static data
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, ["repository", repo_name])
# TODO: Multilist approach. We will obsolete it in future
createJSON (data, destdir+"/"+listname_file+"-rep-static.json")
# Multirepos filename
createJSON (data, destdir+ "/"+listname_file+"-mls-rep-static.json")
def companiesData(period, startdate, enddate, identities_db, destdir, bots, npeople):
companies = SCM.companies_name_wo_affs(bots, startdate, enddate)
companies = companies['name']
createJSON(companies, destdir+"/scm-companies.json")
for company in companies:
company_name = "'"+ company+ "'"
print (company_name)
evol_data = SCM.GetSCMEvolutionaryData(period, startdate, enddate, identities_db, ["company", company_name])
evol_data = completePeriodIds(evol_data)
createJSON(evol_data, destdir+"/"+company+"-scm-com-evolutionary.json")
agg = SCM.GetSCMStaticData(period, startdate, enddate, identities_db, ["company", company_name])
createJSON(agg, destdir+"/"+company+"-scm-com-static.json")
top_authors = SCM.company_top_authors(company_name, startdate, enddate, npeople)
createJSON(top_authors, destdir+"/"+company+"-scm-com-top-authors.json", False)
for i in [2006,2009,2012]:
data = SCM.company_top_authors_year(company_name, i, npeople)
createJSON(data, destdir+"/"+company+"-scm-top-authors_"+str(i)+".json", False)
commits = SCM.GetCommitsSummaryCompanies(period, startdate, enddate, opts.identities_db, 10)
createJSON (commits, destdir+"/scm-companies-commits-summary.json")
def peopleData(period, startdate, enddate, idb, destdir, top_data):
top = safeTopIds(top_data['reviewers'])
top += safeTopIds(top_data['reviewers.last year'])
top += safeTopIds(top_data['reviewers.last month'])
top += safeTopIds(top_data['openers.'])
top += safeTopIds(top_data['openers.last year'])
top += safeTopIds(top_data['openers.last_month'])
top += safeTopIds(top_data['mergers.'])
top += safeTopIds(top_data['mergers.last year'])
top += safeTopIds(top_data['mergers.last_month'])
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir + "/scr-people.json", False)
for upeople_id in people:
# evol = vizr.GetPeopleEvolSCR(upeople_id, period, startdate, enddate)
# evol = completePeriodIds(dataFrame2Dict(evol))
evol = SCR.GetPeopleEvolSCR(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
createJSON(
evol,
destdir + "/people-" + str(upeople_id) + "-scr-evolutionary.json")
# agg = dataFrame2Dict(vizr.GetPeopleStaticSCR(upeople_id, startdate, enddate))
agg = SCR.GetPeopleStaticSCR(upeople_id, startdate, enddate)
createJSON(agg,
destdir + "/people-" + str(upeople_id) + "-scr-static.json")
def companiesData(period, startdate, enddate, identities_db, destdir, npeople):
# companies = valRtoPython(vizr.companiesNames(identities_db, startdate, enddate))
companies = MLS.companiesNames(identities_db, startdate, enddate)
createJSON(companies, destdir + "/mls-companies.json")
for company in companies:
company_name = "'" + company + "'"
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, ["company", company_name])
data = completePeriodIds(data)
if company in ["company4", "Deutsche Telekom", "IBM"]:
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + company + "-mls-com-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + company + "-mls-com-evolutionary.json")
top_senders = MLS.companyTopSenders(company, identities_db, startdate,
enddate, npeople)
createJSON(top_senders,
destdir + "/" + company + "-mls-com-top-senders.json",
False)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, ["company", company_name])
createJSON(data, destdir + "/" + company + "-mls-com-static.json")
sent = MLS.GetSentSummaryCompanies(period, startdate, enddate,
opts.identities_db, 10)
createJSON(sent, opts.destdir + "/mls-sent-companies-summary.json")
def companiesData(period, startdate, enddate, identities_db, destdir, closed_condition, bots, npeople):
# companies = dataFrame2Dict(vizr.GetCompaniesNameITS(startdate, enddate, identities_db, closed_condition, bots))
companies = ITS.GetCompaniesNameITS(startdate, enddate, identities_db, closed_condition, bots)
companies = companies['name']
createJSON(companies, destdir+"/its-companies.json")
for company in companies:
company_name = "'"+ company+ "'"
print (company_name)
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db, ['company', company_name], closed_condition)
evol = completePeriodIds(evol)
if company in ['IBM','Internap']:
createJSON(evol, destdir+"/"+company+"-its-com-evolutionary.json", False)
else:
createJSON(evol, destdir+"/"+company+"-its-com-evolutionary.json")
agg = ITS.AggITSInfo(period, startdate, enddate, identities_db, ['company', company_name], closed_condition)
createJSON(agg, destdir+"/"+company+"-its-com-static.json")
top = ITS.GetCompanyTopClosers(company_name, startdate, enddate, identities_db, bots, closed_condition, npeople)
createJSON(top, destdir+"/"+company+"-its-com-top-closers.json", False)
closed = ITS.GetClosedSummaryCompanies(period, startdate, enddate, identities_db, closed_condition, 10)
createJSON (closed, opts.destdir+"/its-closed-companies-summary.json")
def countriesData(period, startdate, enddate, identities_db, destdir, npeople):
countries = MLS.countriesNames(identities_db, startdate, enddate)
createJSON(countries, destdir + "/mls-countries.json")
for country in countries:
country_name = "'" + country + "'"
type_analysis = ["country", country_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
data = completePeriodIds(data)
if (country == "country5" or country == "country2"):
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + country + "-mls-cou-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + country + "-mls-cou-evolutionary.json")
top_senders = MLS.countryTopSenders(country, identities_db, startdate,
enddate, npeople)
createJSON(top_senders,
destdir + "/" + country + "-mls-cou-top-senders.json")
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
createJSON(data, destdir + "/" + country + "-mls-cou-static.json")
def peopleData(period, startdate, enddate, idb, destdir, top_data):
top = safeTopIds(top_data['reviewers'])
top += safeTopIds(top_data['reviewers.last year'])
top += safeTopIds(top_data['reviewers.last month'])
top += safeTopIds(top_data['openers.'])
top += safeTopIds(top_data['openers.last year'])
top += safeTopIds(top_data['openers.last_month'])
top += safeTopIds(top_data['mergers.'])
top += safeTopIds(top_data['mergers.last year'])
top += safeTopIds(top_data['mergers.last_month'])
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir+"/scr-people.json", False)
for upeople_id in people:
# evol = vizr.GetPeopleEvolSCR(upeople_id, period, startdate, enddate)
# evol = completePeriodIds(dataFrame2Dict(evol))
evol = SCR.GetPeopleEvolSCR(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
createJSON(evol, destdir+"/people-"+str(upeople_id)+"-scr-evolutionary.json")
# agg = dataFrame2Dict(vizr.GetPeopleStaticSCR(upeople_id, startdate, enddate))
agg = SCR.GetPeopleStaticSCR(upeople_id, startdate, enddate)
createJSON(agg, destdir+"/people-"+str(upeople_id)+"-scr-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir, npeople):
domains = MLS.domainsNames(identities_db, startdate, enddate)
createJSON(domains, destdir + "/mls-domains.json")
for domain in domains:
domain_name = "'" + domain + "'"
type_analysis = ["domain", domain_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
data = completePeriodIds(data)
if (domain == "everybody" or domain == "hallowelt"):
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + domain + "-mls-dom-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + domain + "-mls-dom-evolutionary.json")
data = MLS.domainTopSenders(domain, identities_db, startdate, enddate,
npeople)
createJSON(data, destdir + "/" + domain + "-mls-dom-top-senders.json",
False)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
createJSON(data, destdir + "/" + domain + "-mls-dom-static.json")
def ticketsStates(period, startdate, enddate, identities_db, backend):
evol = {}
for status in backend.statuses:
print ("Working with ticket status: " + status)
#Evolution of the backlog
tickets_status = vizr.GetEvolBacklogTickets(period, startdate, enddate, status, backend.name_log_table)
tickets_status = dataFrame2Dict(tickets_status)
tickets_status = completePeriodIds(tickets_status)
# rename key
tickets_status[status] = tickets_status.pop("pending_tickets")
#Issues per status
current_status = vizr.GetCurrentStatus(period, startdate, enddate, identities_db, status)
current_status = completePeriodIds(dataFrame2Dict(current_status))
#Merging data
evol = dict(evol.items() + current_status.items() + tickets_status.items())
return evol
def companiesData(period, startdate, enddate, idb, destdir):
# companies = dataFrame2Dict(vizr.GetCompaniesSCRName(startdate, enddate, idb))
companies = SCR.GetCompaniesSCRName(startdate, enddate, idb)
companies = companies['name']
companies_files = [company.replace('/', '_') for company in companies]
createJSON(companies_files, destdir+"/scr-companies.json")
# missing information from the rest of type of reviews, patches and
# number of patches waiting for reviewer and submitter
for company in companies:
company_file = company.replace("/","_")
type_analysis = ['company', company]
# Evol
evol = {}
# data = vizr.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = SCR.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsMerged(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsAbandoned(period, startdate, enddate, type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
if (period == "month"):
data = SCR.EvolTimeToReviewSCR(period, startdate, enddate, idb, type_analysis)
data['review_time_days_avg'] = checkFloatArray(data['review_time_days_avg'])
data['review_time_days_median'] = checkFloatArray(data['review_time_days_median'])
evol = dict(evol.items() + completePeriodIds(data).items())
createJSON(evol, destdir+ "/"+company_file+"-scr-com-evolutionary.json")
# Static
agg = {}
# data = vizr.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# agg = dict(agg.items() + dataFrame2Dict(data).items())
data = SCR.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsMerged(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsAbandoned(period, startdate, enddate, type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticTimeToReviewSCR(startdate, enddate, idb, type_analysis)
val = data['review_time_days_avg']
if (not val or val == 0): data['review_time_days_avg'] = 0
else: data['review_time_days_avg'] = float(val)
val = data['review_time_days_median']
if (not val or val == 0): data['review_time_days_median'] = 0
else: data['review_time_days_median'] = float(val)
agg = dict(agg.items() + data.items())
createJSON(agg, destdir+"/"+company_file+"-scr-com-static.json")
def GetMLSInfo (period, startdate, enddate, identities_db, rfield, type_analysis, evolutionary):
if (evolutionary == True):
sent = EvolEmailsSent(period, startdate, enddate, identities_db, type_analysis)
sent = completePeriodIds(sent)
senders = EvolMLSSenders(period, startdate, enddate, identities_db, type_analysis)
senders = completePeriodIds(senders)
repositories = EvolMLSRepositories(rfield, period, startdate, enddate, identities_db, type_analysis)
repositories = completePeriodIds(repositories)
threads = EvolThreads(period, startdate, enddate, identities_db, type_analysis)
threads = completePeriodIds(threads)
sent_response = EvolMLSResponses(period, startdate, enddate, identities_db, type_analysis)
sent_response = completePeriodIds(sent_response)
senders_response = EvolMLSSendersResponse(period, startdate, enddate, identities_db, type_analysis)
senders_response = completePeriodIds(senders_response)
senders_init = EvolMLSSendersInit(period, startdate, enddate, identities_db, type_analysis)
senders_init = completePeriodIds(senders_init)
else:
sent = AggEmailsSent(period, startdate, enddate, identities_db, type_analysis)
senders = AggMLSSenders(period, startdate, enddate, identities_db, type_analysis)
repositories = AggMLSRepositories(rfield, period, startdate, enddate, identities_db, type_analysis)
threads = AggThreads(period, startdate, enddate, identities_db, type_analysis)
sent_response = AggMLSResponses(period, startdate, enddate, identities_db, type_analysis)
senders_response = AggMLSSendersResponse(period, startdate, enddate, identities_db, type_analysis)
senders_init = AggMLSSendersInit(period, startdate, enddate, identities_db, type_analysis)
data = dict(sent.items() + senders.items()+ repositories.items())
data = dict(data.items() + threads.items()+ sent_response.items())
data = dict(data.items() + senders_response.items() + senders_init.items())
return (data)
def countriesData(period, startdate, enddate, idb, destdir):
# countries = dataFrame2Dict(vizr.GetCountriesSCRName(startdate, enddate, idb))
countries = SCR.GetCountriesSCRName(startdate, enddate, idb)
countries = countries['name']
countries_files = [country.replace('/', '_') for country in countries]
createJSON(countries_files, destdir + "/scr-countries.json")
# missing information from the rest of type of reviews, patches and
# number of patches waiting for reviewer and submitter
for country in countries:
country_file = country.replace("/", "_")
type_analysis = ['country', country]
# Evol
evol = {}
# data = vizr.EvolReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = SCR.EvolReviewsSubmitted(period, startdate, enddate,
type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsMerged(period, startdate, enddate, type_analysis,
idb)
evol = dict(evol.items() + completePeriodIds(data).items())
data = SCR.EvolReviewsAbandoned(period, startdate, enddate,
type_analysis, idb)
evol = dict(evol.items() + completePeriodIds(data).items())
# TODO: when empty abandoned does not appeat at all in R JSON
createJSON(evol,
destdir + "/" + country_file + "-scr-cou-evolutionary.json",
False)
# Static
agg = {}
# data = vizr.StaticReviewsSubmitted(period, startdate, enddate, type_analysis, idb)
# agg = dict(agg.items() + dataFrame2Dict(data).items())
data = SCR.StaticReviewsSubmitted(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsMerged(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
data = SCR.StaticReviewsAbandoned(period, startdate, enddate,
type_analysis, idb)
agg = dict(agg.items() + data.items())
createJSON(agg, destdir + "/" + country_file + "-scr-cou-static.json")
def tsData(period, startdate, enddate, identities_db, destdir, granularity, conf):
# data = vizr.GetSCMEvolutionaryData(period, startdate, enddate, identities_db)
# evol_data = completePeriodIds(dataFrame2Dict(data))
data = SCM.GetSCMEvolutionaryData(period, startdate, enddate, identities_db, None)
evol_data = completePeriodIds(data)
if ('companies' in reports) :
data = SCM.EvolCompanies(period, startdate, enddate)
evol_data = dict(evol_data.items() + completePeriodIds(data).items())
if ('countries' in reports) :
data = SCM.EvolCountries(period, startdate, enddate)
evol_data = dict(evol_data.items() + completePeriodIds(data).items())
if ('domains' in reports) :
data = SCM.EvolDomains(period, startdate, enddate)
evol_data = dict(evol_data.items() + completePeriodIds(data).items())
createJSON (evol_data, destdir+"/scm-evolutionary.json")
def ticketsStates(period, startdate, enddate, identities_db, backend):
evol = {}
for status in backend.statuses:
print("Working with ticket status: " + status)
#Evolution of the backlog
tickets_status = vizr.GetEvolBacklogTickets(period, startdate, enddate,
status,
backend.name_log_table)
tickets_status = dataFrame2Dict(tickets_status)
tickets_status = completePeriodIds(tickets_status)
# rename key
tickets_status[status] = tickets_status.pop("pending_tickets")
#Issues per status
current_status = vizr.GetCurrentStatus(period, startdate, enddate,
identities_db, status)
current_status = completePeriodIds(dataFrame2Dict(current_status))
#Merging data
evol = dict(evol.items() + current_status.items() +
tickets_status.items())
return evol
def companies_countriesData(period, startdate, enddate, identities_db, destdir):
companies = SCM.companies_name(startdate, enddate)
companies = companies['name']
for company in companies:
company_name = "'"+company+ "'"
countries = SCM.scm_countries_names(identities_db,startdate, enddate)
countries = countries['name']
for country in countries :
print (country, "=>", company)
data = SCM.scm_companies_countries_evol(identities_db, company, country, nperiod, startdate, enddate)
data = completePeriodIds(data)
createJSON (data, destdir + "/"+company+"_"+country+"-scm-evolutionary.json", False)
def GetITSInfo(period, startdate, enddate, identities_db, type_analysis,
closed_condition, evolutionary):
# Meta function to aggregate all of the evolutionary or
# aggregated functions
data = {}
if (evolutionary):
closed = EvolIssuesClosed(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
closed = completePeriodIds(closed)
closers = EvolIssuesClosers(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
closers = completePeriodIds(closers)
changed = EvolIssuesChanged(period, startdate, enddate, identities_db,
type_analysis)
changed = completePeriodIds(changed)
changers = EvolIssuesChangers(period, startdate, enddate,
identities_db, type_analysis)
changers = completePeriodIds(changers)
open = EvolIssuesOpened(period, startdate, enddate, identities_db,
type_analysis)
open = completePeriodIds(open)
openers = EvolIssuesOpeners(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
openers = completePeriodIds(openers)
repos = EvolIssuesRepositories(period, startdate, enddate,
identities_db, type_analysis)
repos = completePeriodIds(repos)
else:
closed = AggIssuesClosed(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
closers = AggIssuesClosers(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
changed = AggIssuesChanged(period, startdate, enddate, identities_db,
type_analysis)
changers = AggIssuesChangers(period, startdate, enddate, identities_db,
type_analysis)
open = AggIssuesOpened(period, startdate, enddate, identities_db,
type_analysis)
openers = AggIssuesOpeners(period, startdate, enddate, identities_db,
type_analysis, closed_condition)
repos = AggIssuesRepositories(period, startdate, enddate,
identities_db, type_analysis)
init_date = GetInitDate(startdate, enddate, identities_db,
type_analysis)
end_date = GetEndDate(startdate, enddate, identities_db, type_analysis)
data = dict(closed.items() + closers.items() + changed.items())
data = dict(data.items() + changers.items() + open.items())
data = dict(data.items() + openers.items() + repos.items())
if (not evolutionary):
data = dict(data.items() + init_date.items() + end_date.items())
return (data)
def tsData(period, startdate, enddate, identities_db, destdir, granularity,
conf):
evol = {}
# data = vizr.EvolMLSInfo(period, startdate, enddate, identities_db, rfield)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('companies' in reports):
data = MLS.EvolMLSCompanies(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('countries' in reports):
data = MLS.EvolMLSCountries(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('domains' in reports):
data = MLS.EvolMLSDomains(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
createJSON(evol, destdir + "/mls-evolutionary.json")
def tsData(period, startdate, enddate, identities_db, destdir, granularity, conf):
evol = {}
# data = vizr.EvolMLSInfo(period, startdate, enddate, identities_db, rfield)
# evol = dict(evol.items() + completePeriodIds(dataFrame2Dict(data)).items())
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('companies' in reports):
data = MLS.EvolMLSCompanies(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('countries' in reports):
data = MLS.EvolMLSCountries(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
if ('domains' in reports):
data = MLS.EvolMLSDomains(period, startdate, enddate, identities_db)
evol = dict(evol.items() + completePeriodIds(data).items())
createJSON (evol, destdir+"/mls-evolutionary.json")
def countriesData(period, startdate, enddate, identities_db, destdir):
countries = SCM.scm_countries_names(identities_db,startdate, enddate)
countries = countries['name']
createJSON(countries, destdir+"/scm-countries.json")
for country in countries:
print (country)
country_name = "'"+country+"'"
evol_data = SCM.GetSCMEvolutionaryData(period, startdate, enddate, identities_db, ["country", country_name])
evol_data = completePeriodIds(evol_data)
createJSON (evol_data, destdir+"/"+country+"-scm-cou-evolutionary.json")
agg = SCM.GetSCMStaticData(period, startdate, enddate, identities_db, ["country", country_name])
createJSON (agg, destdir+"/"+country+"-scm-cou-static.json")
def peopleData(period, startdate, enddate, identities_db, destdir, top_data):
top = top_data['senders.']["id"]
top += top_data['senders.last year']["id"]
top += top_data['senders.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir+"/mls-people.json", False)
for upeople_id in people:
evol = MLS.GetEvolPeopleMLS(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
createJSON(evol, destdir+"/people-"+str(upeople_id)+"-mls-evolutionary.json")
static = MLS.GetStaticPeopleMLS(upeople_id, startdate, enddate)
createJSON(static, destdir+"/people-"+str(upeople_id)+"-mls-static.json")
def countriesData(period, startdate, enddate, identities_db, destdir, closed_condition):
# countries = dataFrame2Dict(vizr.GetCountriesNamesITS(startdate, enddate, identities_db, closed_condition))
countries = ITS.GetCountriesNamesITS(startdate, enddate, identities_db, closed_condition)
countries = countries['name']
createJSON(countries, destdir+"/its-countries.json")
for country in countries :
print (country)
country_name = "'" + country + "'"
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db, ['country', country_name], closed_condition)
evol = completePeriodIds(evol)
createJSON (evol, destdir+"/"+country+"-its-cou-evolutionary.json")
data = ITS.AggITSInfo(period, startdate, enddate, identities_db, ['country', country_name], closed_condition)
createJSON (data, destdir+"/"+country+"-its-cou-static.json")
def reposData(period, startdate, enddate, identities_db, destdir, conf,
repofield, npeople):
repos = MLS.reposNames(rfield, startdate, enddate)
createJSON(repos, destdir + "/mls-lists.json")
repos = repos['mailing_list_url']
check = True
if not isinstance(repos, (list)):
repos = [repos]
check = False
repos_files = [
repo.replace('/', '_').replace("<", "__").replace(">", "___")
for repo in repos
]
createJSON(repos_files, destdir + "/mls-repos.json", check)
for repo in repos:
# Evol data
repo_name = "'" + repo + "'"
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, ["repository", repo_name])
data = completePeriodIds(data)
listname_file = repo.replace("/",
"_").replace("<",
"__").replace(">", "___")
# TODO: Multilist approach. We will obsolete it in future
createJSON(
data, destdir + "/mls-" + listname_file + "-rep-evolutionary.json")
# Multirepos filename
createJSON(
data, destdir + "/" + listname_file + "-mls-rep-evolutionary.json")
top_senders = MLS.repoTopSenders(repo, identities_db, startdate,
enddate, repofield, npeople)
createJSON(top_senders,
destdir + "/" + listname_file + "-mls-rep-top-senders.json",
False)
# Static data
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, ["repository", repo_name])
# TODO: Multilist approach. We will obsolete it in future
createJSON(data, destdir + "/" + listname_file + "-rep-static.json")
# Multirepos filename
createJSON(data,
destdir + "/" + listname_file + "-mls-rep-static.json")
def reposData(period, startdate, enddate, idb, destdir):
# repos = valRtoPython(vizr.GetReposNameIRC())
repos = IRC.GetReposNameIRC()
repos_file = destdir+"/irc-repos.json"
createJSON(repos, repos_file)
for repo in repos:
# evol = vizr.GetRepoEvolSentSendersIRC(repo, period, startdate, enddate)
evol = IRC.GetRepoEvolSentSendersIRC(repo, period, startdate, enddate)
# evol = completePeriodIds(dataFrame2Dict(evol))
evol = completePeriodIds(evol)
repo_file = destdir+"/"+repo+"-irc-rep-evolutionary.json"
createJSON(evol, repo_file)
repo_file = destdir+"/"+repo+"-irc-rep-static.json"
# aggdata = dataFrame2Dict(vizr.GetRepoStaticSentSendersIRC(repo, startdate, enddate))
aggdata = IRC.GetRepoStaticSentSendersIRC(repo, startdate, enddate)
createJSON(aggdata, repo_file)
def reposData(period, startdate, enddate, idb, destdir):
# repos = valRtoPython(vizr.GetReposNameIRC())
repos = IRC.GetReposNameIRC()
repos_file = destdir + "/irc-repos.json"
createJSON(repos, repos_file)
for repo in repos:
# evol = vizr.GetRepoEvolSentSendersIRC(repo, period, startdate, enddate)
evol = IRC.GetRepoEvolSentSendersIRC(repo, period, startdate, enddate)
# evol = completePeriodIds(dataFrame2Dict(evol))
evol = completePeriodIds(evol)
repo_file = destdir + "/" + repo + "-irc-rep-evolutionary.json"
createJSON(evol, repo_file)
repo_file = destdir + "/" + repo + "-irc-rep-static.json"
# aggdata = dataFrame2Dict(vizr.GetRepoStaticSentSendersIRC(repo, startdate, enddate))
aggdata = IRC.GetRepoStaticSentSendersIRC(repo, startdate, enddate)
createJSON(aggdata, repo_file)
def peopleData(period, startdate, enddate, identities_db, destdir, top_authors_data):
top = top_authors_data['authors.']["id"]
top += top_authors_data['authors.last year']["id"]
top += top_authors_data['authors.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir+"/scm-people.json", False)
for upeople_id in people :
evol_data = SCM.GetEvolPeopleSCM(upeople_id, period, startdate, enddate)
evol_data = completePeriodIds(evol_data)
createJSON (evol_data, destdir+"/people-"+str(upeople_id)+"-scm-evolutionary.json")
agg = SCM.GetStaticPeopleSCM(upeople_id, startdate, enddate)
createJSON (agg, destdir+"/people-"+str(upeople_id)+"-scm-static.json")
pass
def GetMLSInfo(period, startdate, enddate, identities_db, rfield,
type_analysis, evolutionary):
if (evolutionary == True):
sent = EvolEmailsSent(period, startdate, enddate, identities_db,
type_analysis)
sent = completePeriodIds(sent)
senders = EvolMLSSenders(period, startdate, enddate, identities_db,
type_analysis)
senders = completePeriodIds(senders)
repositories = EvolMLSRepositories(rfield, period, startdate, enddate,
identities_db, type_analysis)
repositories = completePeriodIds(repositories)
threads = EvolThreads(period, startdate, enddate, identities_db,
type_analysis)
threads = completePeriodIds(threads)
sent_response = EvolMLSResponses(period, startdate, enddate,
identities_db, type_analysis)
sent_response = completePeriodIds(sent_response)
senders_response = EvolMLSSendersResponse(period, startdate, enddate,
identities_db, type_analysis)
senders_response = completePeriodIds(senders_response)
senders_init = EvolMLSSendersInit(period, startdate, enddate,
identities_db, type_analysis)
senders_init = completePeriodIds(senders_init)
else:
sent = AggEmailsSent(period, startdate, enddate, identities_db,
type_analysis)
senders = AggMLSSenders(period, startdate, enddate, identities_db,
type_analysis)
repositories = AggMLSRepositories(rfield, period, startdate, enddate,
identities_db, type_analysis)
threads = AggThreads(period, startdate, enddate, identities_db,
type_analysis)
sent_response = AggMLSResponses(period, startdate, enddate,
identities_db, type_analysis)
senders_response = AggMLSSendersResponse(period, startdate, enddate,
identities_db, type_analysis)
senders_init = AggMLSSendersInit(period, startdate, enddate,
identities_db, type_analysis)
data = dict(sent.items() + senders.items() + repositories.items())
data = dict(data.items() + threads.items() + sent_response.items())
data = dict(data.items() + senders_response.items() + senders_init.items())
return (data)
def peopleData(period, startdate, enddate, identities_db, destdir, top_data):
top = top_data['senders.']["id"]
top += top_data['senders.last year']["id"]
top += top_data['senders.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir + "/mls-people.json", False)
for upeople_id in people:
evol = MLS.GetEvolPeopleMLS(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
createJSON(
evol,
destdir + "/people-" + str(upeople_id) + "-mls-evolutionary.json")
static = MLS.GetStaticPeopleMLS(upeople_id, startdate, enddate)
createJSON(static,
destdir + "/people-" + str(upeople_id) + "-mls-static.json")
def peopleData(period, startdate, enddate, identities_db, destdir, closed_condition, top_data):
top = top_data['closers.']["id"]
top += top_data['closers.last year']["id"]
top += top_data['closers.last month']["id"]
top += top_data['openers.']["id"]
top += top_data['openers.last year']["id"]
top += top_data['openers.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir+"/its-people.json", False)
for upeople_id in people :
evol = ITS.GetPeopleEvolITS(upeople_id, period, startdate, enddate, closed_condition)
evol = completePeriodIds(evol)
createJSON (evol, destdir+"/people-"+str(upeople_id)+"-its-evolutionary.json")
data = ITS.GetPeopleStaticITS(upeople_id, startdate, enddate, closed_condition)
createJSON (data, destdir+"/people-"+str(upeople_id)+"-its-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir, closed_condition, bots, npeople):
# domains = dataFrame2Dict(vizr.GetDomainsNameITS(startdate, enddate, identities_db, closed_condition, bots))
domains = ITS.GetDomainsNameITS(startdate, enddate, identities_db, closed_condition, bots)
domains = domains['name']
createJSON(domains, destdir+"/its-domains.json")
for domain in domains:
domain_name = "'"+ domain + "'"
print (domain_name)
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db, ['domain', domain_name], closed_condition)
evol = completePeriodIds(evol)
createJSON(evol, destdir+"/"+domain+"-its-dom-evolutionary.json")
agg = ITS.AggITSInfo(period, startdate, enddate, identities_db, ['domain', domain_name], closed_condition)
createJSON(agg, destdir+"/"+domain+"-its-dom-static.json")
top = ITS.GetDomainTopClosers(domain_name, startdate, enddate, identities_db, bots, closed_condition, npeople)
createJSON(top, destdir+"/"+domain+"-its-dom-top-closers.json", False)
def peopleData(period, startdate, enddate, idb, destdir, top_data):
top = top_data['senders.']["id"]
top += top_data['senders.last year']["id"]
top += top_data['senders.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir+"/irc-people.json", False)
for upeople_id in people:
# evol = dataFrame2Dict(vizr.GetEvolPeopleIRC(upeople_id, period, startdate, enddate))
evol = IRC.GetEvolPeopleIRC(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
person_file = destdir+"/people-"+str(upeople_id)+"-irc-evolutionary.json"
createJSON(evol, person_file)
person_file = destdir+"/people-"+str(upeople_id)+"-irc-static.json"
# aggdata = dataFrame2Dict(vizr.GetStaticPeopleIRC(upeople_id, startdate, enddate))
aggdata = IRC.GetStaticPeopleIRC(upeople_id, startdate, enddate)
createJSON(aggdata, person_file)
def domainsData(period, startdate, enddate, identities_db, destdir):
domains = SCM.scm_domains_names(identities_db,startdate, enddate)
domains = domains['name']
createJSON(domains, destdir+"/scm-domains.json")
# Some R ts are wrong
bad_R_json_domains = ['gerrit','gmx','emsenhuber','bitergia']
for domain in domains :
domain_name = "'"+domain+"'"
print (domain_name)
evol_data = SCM.GetSCMEvolutionaryData(period, startdate, enddate, identities_db, ["domain", domain_name])
evol_data = completePeriodIds(evol_data)
if domain in bad_R_json_domains:
createJSON(evol_data, destdir+"/"+domain+"-scm-dom-evolutionary.json", False)
else:
createJSON(evol_data, destdir+"/"+domain+"-scm-dom-evolutionary.json")
agg = SCM.GetSCMStaticData(period, startdate, enddate, identities_db, ["domain", domain_name])
createJSON(agg, destdir+ "/"+domain+"-scm-dom-static.json")
def reposData(period, startdate, enddate, identities_db, destdir, conf):
# repos = dataFrame2Dict(vizr.repos_name(startdate, enddate))
repos = SCM.repos_name(startdate, enddate)
repos = repos['name']
if not isinstance(repos, (list)):
repos = [repos]
createJSON(repos, destdir+"/scm-repos.json", False)
else:
createJSON(repos, destdir+"/scm-repos.json")
for repo in repos :
repo_name = "'"+ repo+ "'"
print (repo_name)
evol_data = SCM.GetSCMEvolutionaryData(period, startdate, enddate, identities_db, ["repository", repo_name])
evol_data = completePeriodIds(evol_data)
createJSON(evol_data, destdir+"/"+repo+"-scm-rep-evolutionary.json")
agg = SCM.GetSCMStaticData(period, startdate, enddate, identities_db, ["repository", repo_name])
createJSON(agg, destdir+"/"+repo+"-scm-rep-static.json")
def countriesData(period, startdate, enddate, identities_db, destdir, npeople):
countries = MLS.countriesNames(identities_db, startdate, enddate)
createJSON (countries, destdir + "/mls-countries.json")
for country in countries:
country_name = "'" + country + "'"
type_analysis = ["country", country_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis)
data = completePeriodIds(data)
if (country == "country5" or country == "country2"):
# Wrong JSON generated in R. Don't check
createJSON(data, destdir+"/"+country+"-mls-cou-evolutionary.json", False)
else:
createJSON (data, destdir+"/"+country+"-mls-cou-evolutionary.json")
top_senders = MLS.countryTopSenders (country, identities_db, startdate, enddate, npeople)
createJSON(top_senders, destdir+"/"+country+"-mls-cou-top-senders.json")
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis)
createJSON (data, destdir+"/"+country+"-mls-cou-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir, npeople):
domains = MLS.domainsNames(identities_db, startdate, enddate)
createJSON(domains, destdir+"/mls-domains.json")
for domain in domains:
domain_name = "'"+domain+"'"
type_analysis = ["domain", domain_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis)
data = completePeriodIds(data)
if (domain == "everybody" or domain == "hallowelt"):
# Wrong JSON generated in R. Don't check
createJSON(data, destdir+"/"+domain+"-mls-dom-evolutionary.json", False)
else:
createJSON(data, destdir+"/"+domain+"-mls-dom-evolutionary.json")
data = MLS.domainTopSenders(domain, identities_db, startdate, enddate, npeople)
createJSON(data, destdir+"/"+domain+"-mls-dom-top-senders.json", False)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis)
createJSON(data, destdir+"/"+domain+"-mls-dom-static.json")
def countriesData(period, startdate, enddate, identities_db, destdir,
closed_condition):
# countries = dataFrame2Dict(vizr.GetCountriesNamesITS(startdate, enddate, identities_db, closed_condition))
countries = ITS.GetCountriesNamesITS(startdate, enddate, identities_db,
closed_condition)
countries = countries['name']
createJSON(countries, destdir + "/its-countries.json")
for country in countries:
print(country)
country_name = "'" + country + "'"
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db,
['country', country_name], closed_condition)
evol = completePeriodIds(evol)
createJSON(evol,
destdir + "/" + country + "-its-cou-evolutionary.json")
data = ITS.AggITSInfo(period, startdate, enddate, identities_db,
['country', country_name], closed_condition)
createJSON(data, destdir + "/" + country + "-its-cou-static.json")
def peopleData(period, startdate, enddate, idb, destdir, top_data):
top = top_data['senders.']["id"]
top += top_data['senders.last year']["id"]
top += top_data['senders.last month']["id"]
# remove duplicates
people = list(set(top))
# the order is not the same than in R json
createJSON(people, destdir + "/irc-people.json", False)
for upeople_id in people:
# evol = dataFrame2Dict(vizr.GetEvolPeopleIRC(upeople_id, period, startdate, enddate))
evol = IRC.GetEvolPeopleIRC(upeople_id, period, startdate, enddate)
evol = completePeriodIds(evol)
person_file = destdir + "/people-" + str(
upeople_id) + "-irc-evolutionary.json"
createJSON(evol, person_file)
person_file = destdir + "/people-" + str(
upeople_id) + "-irc-static.json"
# aggdata = dataFrame2Dict(vizr.GetStaticPeopleIRC(upeople_id, startdate, enddate))
aggdata = IRC.GetStaticPeopleIRC(upeople_id, startdate, enddate)
createJSON(aggdata, person_file)
def reposData(period, startdate, enddate, identities_db, destdir, conf, closed_condition):
# repos = dataFrame2Dict(vizr.GetReposNameITS(startdate, enddate))
repos = ITS.GetReposNameITS(startdate, enddate)
repos = repos['name']
if not isinstance(repos, (list)):
repos = [repos]
createJSON(repos, destdir+"/its-repos.json", False)
else:
createJSON(repos, destdir+"/its-repos.json")
for repo in repos :
repo_name = "'"+ repo+ "'"
repo_file = repo.replace("/","_")
evol = ITS.EvolITSInfo(period, startdate, enddate, identities_db, ['repository', repo_name], closed_condition)
evol = completePeriodIds(evol)
if (repo_file == "http:__tracker.ceph.com_projects_rados-java_"):
createJSON(evol, destdir+"/"+repo_file+"-its-rep-evolutionary.json", False)
else:
createJSON(evol, destdir+"/"+repo_file+"-its-rep-evolutionary.json")
agg = ITS.AggITSInfo(period, startdate, enddate, identities_db, ['repository', repo_name], closed_condition)
createJSON(agg, destdir+"/"+repo_file+"-its-rep-static.json")