def get_top_data_authors(startdate, enddate, i_db, filter_, npeople):
top = {}
mauthors = DataSource.get_metrics("authors", SCM)
if mauthors is None: return top
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
mfilter.global_filter = mauthors.filters.global_filter
if filter_ is None:
top['authors.'] = mauthors.get_list(mfilter, 0)
top['authors.last month'] = mauthors.get_list(mfilter, 31)
top['authors.last year'] = mauthors.get_list(mfilter, 365)
elif filter_.get_name() in ["company", "repository", "project"]:
if filter_.get_name() in ["company", "repository", "project"]:
top['authors.'] = mauthors.get_list(mfilter, 0)
top['authors.last month'] = mauthors.get_list(mfilter, 31)
top['authors.last year'] = mauthors.get_list(mfilter, 365)
else:
# If we have performance issues with tops, remove filters above
# to avoid computing trends for tops
top = mauthors.get_list(mfilter)
else:
logging.info("Top authors not support for " + filter_.get_name())
return top
def get_trends(self, date, days):
""" Returns the trend metrics between now and now-days values """
if self.filters.type_analysis and self.filters.type_analysis[1] is None:
return self._get_trends_all_items(date, days)
# Keeping state of origin filters
filters = self.filters
chardates = GetDates(date, days)
self.filters = MetricFilters(filters.period,
chardates[1], chardates[0], filters.type_analysis)
self.filters.global_filter = filters.global_filter
self.filters.closed_condition = filters.closed_condition
last = self.get_agg()
if last[self.id] is not None:
last = int(last[self.id])
else: last = 0
self.filters = MetricFilters(filters.period,
chardates[2], chardates[1], filters.type_analysis)
self.filters.global_filter = filters.global_filter
self.filters.closed_condition = filters.closed_condition
prev = self.get_agg()
if prev[self.id] is not None:
prev = int(prev[self.id])
else: prev = 0
data = {}
data['diff_net'+self.id+'_'+str(days)] = last - prev
data['percentage_'+self.id+'_'+str(days)] = GetPercentageDiff(prev, last)
data[self.id+'_'+str(days)] = last
# Returning filters to their original value
self.filters = filters
return (data)
def get_top_data_authors (startdate, enddate, i_db, filter_, npeople):
top = {}
mauthors = DataSource.get_metrics("authors", SCM)
if mauthors is None: return top
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
mfilter.global_filter = mauthors.filters.global_filter
if filter_ is None:
top['authors.'] = mauthors.get_list(mfilter, 0)
top['authors.last month'] = mauthors.get_list(mfilter, 31)
top['authors.last year'] = mauthors.get_list(mfilter, 365)
elif filter_.get_name() in ["company","repository","project"]:
if filter_.get_name() in ["company","repository","project"]:
top['authors.'] = mauthors.get_list(mfilter, 0)
top['authors.last month'] = mauthors.get_list(mfilter, 31)
top['authors.last year'] = mauthors.get_list(mfilter, 365)
else:
# If we have performance issues with tops, remove filters above
# to avoid computing trends for tops
top = mauthors.get_list(mfilter)
else:
logging.info("Top authors not support for " + filter_.get_name())
return top
def test_scm_simple(self):
correct = """
{'aging': ActorsDuration:
{'age': datetime.timedelta(13, 3349), 'id': u'6872b4eeab3d91e5f47f142137b6672f64dc5102', 'name': u'6872b4eeab3d91e5f47f142137b6672f64dc5102'}
{'age': datetime.timedelta(30, 14598), 'id': u'86600750577bd979bcf1744a5d8aa387e3c0e088', 'name': u'86600750577bd979bcf1744a5d8aa387e3c0e088'}
{'age': datetime.timedelta(16, 1472), 'id': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1', 'name': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1'}
{'age': datetime.timedelta(29, 12176), 'id': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e', 'name': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e'}
, 'birth': ActorsDuration:
{'age': datetime.timedelta(13, 3349), 'id': u'6872b4eeab3d91e5f47f142137b6672f64dc5102', 'name': u'6872b4eeab3d91e5f47f142137b6672f64dc5102'}
{'age': datetime.timedelta(30, 14598), 'id': u'86600750577bd979bcf1744a5d8aa387e3c0e088', 'name': u'86600750577bd979bcf1744a5d8aa387e3c0e088'}
{'age': datetime.timedelta(16, 1472), 'id': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1', 'name': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1'}
{'age': datetime.timedelta(29, 12176), 'id': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e', 'name': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e'}
}
"""
correct = ''.join(correct.split())
filters = MetricFilters("months", "'2013-12-01'", "'2014-01-01'", [])
dbcon = DSQuery(user="******",
password="******",
database=db_cvsanaly,
identities_db=db_identities)
ages = Ages(dbcon, filters)
result = ages.result(SCM)
self.assertEqual(''.join(str(result).split()), correct)
def releases_info(startdate, enddate, project, opts, people_out, affs_out):
# Releases information.
data = {}
filters = MetricFilters("month", startdate, enddate,
["project", str(project)], opts.npeople,
people_out, affs_out)
# SCM report
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
dataset = scm_report(scm_dbcon, filters)
data["scm"] = dataset
#ITS report
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho,
opts.dbidentities, opts.dbprojects)
dataset = its_report(its_dbcon, filters)
data["its"] = dataset
#SCR Report
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview,
opts.dbidentities, opts.dbprojects)
dataset = scr_report(scr_dbcon, filters)
data["scr"] = dataset
return data
def integrated_projects_activity(dbcon, opts, people_out, affs_out):
# Commits per integrated project
projects = integrated_projects(dbcon)
projects_ids = projects["subproject_id"]
projects_list = []
commits_list = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'" + releases[0] + "'"
enddate = "'" + releases[1] + "'"
for project_id in projects_ids:
project_title = "'" + project_id + "'"
type_analysis = ["project", project_title]
project_filters = MetricFilters(period, startdate, enddate,
type_analysis, 10, people_out,
affs_out)
commits = scm.Commits(dbcon, project_filters)
projects_list.append(project_id)
commits_list.append(int(commits.get_agg()["commits"]))
createCSV({
"projects": projects_list,
"commits": commits_list
}, "./release/integrated_projects_commits.csv")
def integrated_projects_top_contributors(scm_dbcon, people_out, affs_out):
# Top contributor per integrated project
projects = integrated_projects(scm_dbcon)
projects_ids = projects["subproject_id"]
projects_list = []
commits = []
top_contributors = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'" + releases[0] + "'"
enddate = "'" + releases[1] + "'"
for project_id in projects_ids:
project_title = "'" + project_id + "'"
type_analysis = ["project", project_title]
project_filters = MetricFilters(period, startdate, enddate,
type_analysis, 10, people_out,
affs_out)
authors = scm.Authors(scm_dbcon, project_filters)
activity = authors.get_list()
projects_list.append(project_id)
commits.append(activity["commits"][0])
top_contributors.append(activity["authors"][0])
createCSV(
{
"projects": projects_list,
"commits": commits,
"contributors": top_contributors
}, "./release/integrated_projects_top_contributors.csv")
def create_reports_studies(period, startdate, enddate, destdir):
from vizgrimoire.metrics.metrics_filter import MetricFilters
db_identities = Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
studies = Report.get_studies()
metric_filters = MetricFilters(period, startdate, enddate, [])
for ds in get_enabled_data_sources():
ds_dbname = ds.get_db_name()
dbname = Report.get_config()['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
# logging.info(ds.get_name() + " studies active " + str(studies))
for study in studies:
logging.info("Creating report for " + study.id + " for " +
ds.get_name())
try:
obj = study(dbcon, metric_filters)
obj.create_report(ds, destdir)
except TypeError:
import traceback
logging.info(study.id +
" does no support standard API. Not used.")
traceback.print_exc(file=sys.stdout)
continue
def integrated_projects_top_orgs(dbcon, people_out, affs_out):
# Top orgs contributing to each integrated project
# The two first companies are depicted.
projects = integrated_projects(dbcon)
projects_ids = projects["subproject_id"]
projects_list = []
commits_top1 = []
commits_top2 = []
orgs_top1 = []
orgs_top2 = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'"+releases[0]+"'"
enddate = "'"+releases[1]+"'"
for project_id in projects_ids:
project_title = "'" + project_id + "'"
type_analysis = ["project", project_title]
project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10,
people_out, affs_out)
companies = scm.Companies(dbcon, project_filters)
activity = companies.get_list()
projects_list.append(project_id)
commits_top1.append(activity["company_commits"][0])
commits_top2.append(activity["company_commits"][1])
orgs_top1.append(activity["name"][0])
orgs_top2.append(activity["name"][1])
createCSV({"projects":projects_list, "commitstopone":commits_top1, "commitstoptwo":commits_top2, "orgstopone":orgs_top1, "orgstoptwo":orgs_top2}, "./release/integrated_projects_top_orgs.csv")
def timezone_analysis(opts):
from vizgrimoire.analysis.timezone import Timezone
from vizgrimoire.SCM import SCM
from vizgrimoire.MLS import MLS
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly,
opts.dbidentities, opts.dbprojects)
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats,
opts.dbidentities, opts.dbprojects)
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'" + releases[0] + "'"
enddate = "'" + releases[1] + "'"
filters = MetricFilters(period, startdate, enddate, [], 10, "", "")
tz = Timezone(scm_dbcon, filters)
dataset = tz.result(SCM)
labels = dataset["tz"]
commits = dataset["commits"]
authors = dataset["authors"]
bar_chart("Timezone git activity", labels, commits, "commits_tz", authors,
["commits", "authors"])
tz = Timezone(mls_dbcon, filters)
dataset = tz.result(MLS)
messages = dataset["messages"]
authors = dataset["authors"]
bar_chart("Timezone mailing list activity", labels, messages,
"messages_tz", authors, ["messages", "authors"])
def get_top_people(startdate, enddate, idb):
"""Top people for all data sources."""
import vizgrimoire.GrimoireSQL
from vizgrimoire.SCR import SCR
from vizgrimoire.MLS import MLS
from vizgrimoire.ITS import ITS
from vizgrimoire.IRC import IRC
from vizgrimoire.Mediawiki import Mediawiki
from vizgrimoire.metrics.metrics_filter import MetricFilters
from vizgrimoire.data_source import DataSource
npeople = "10000" # max limit, all people included
min_data_sources = 3 # min data sources to be in the list
tops = {}
all_top = {}
all_top_min_ds = {}
period = None
type_analysis = None
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
# SCR and SCM are the same. Don't use both for Tops
mopeners = DataSource.get_metrics("submitters", SCR)
if mopeners:
tops["scr"] = mopeners.get_list(mfilter, 0)
tops["scr"]["identifier"] = tops["scr"].pop("openers")
msenders = DataSource.get_metrics("senders", MLS)
if msenders:
tops["mls"] = msenders.get_list(mfilter, 0)
tops["mls"]["identifier"] = tops["mls"].pop("senders")
mopeners = DataSource.get_metrics("openers", ITS)
if mopeners:
tops["its"] = mopeners.get_list(mfilter, 0)
tops["its"]["identifier"] = tops["its"].pop("openers")
msenders = DataSource.get_metrics("senders", IRC)
if msenders:
tops["irc"] = msenders.get_list(mfilter, 0)
tops["irc"]["identifier"] = tops["irc"].pop("senders")
mauthors = DataSource.get_metrics("authors", Mediawiki)
if mauthors:
tops["mediawiki"] = mauthors.get_list(mfilter, 0)
tops["mediawiki"]["identifier"] = tops["mediawiki"].pop("reviews")
# Build the consolidated top list using all data sources data
# Only people in all data sources is used
for ds in tops:
pos = 1
for id in tops[ds]['id']:
if id not in all_top: all_top[id] = []
all_top[id].append({
"ds": ds,
"pos": pos,
"identifier": tops[ds]['identifier'][pos - 1]
})
pos += 1
for id in all_top:
if len(all_top[id]) >= min_data_sources:
all_top_min_ds[id] = all_top[id]
return all_top_min_ds
def get_top_data (startdate, enddate, identities_db, filter_, npeople):
bots = SCR.get_bots()
top_all = None
mreviewers = DataSource.get_metrics("reviewers", SCR)
mopeners = DataSource.get_metrics("submitters", SCR)
mmergers = DataSource.get_metrics("closers", SCR)
mcorereviewers = DataSource.get_metrics("active_core_reviewers", SCR)
mparticipants = DataSource.get_metrics("participants", SCR)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
if filter_ is None:
top_reviewers = {}
top_reviewers['reviewers'] = mreviewers.get_list(mfilter, 0)
top_reviewers['reviewers.last month']= mreviewers.get_list(mfilter, 31)
top_reviewers['reviewers.last year']= mreviewers.get_list(mfilter, 365)
top_openers = {}
top_openers['openers.'] = mopeners.get_list(mfilter, 0)
top_openers['openers.last month']= mopeners.get_list(mfilter, 31)
top_openers['openers.last year'] = mopeners.get_list(mfilter, 365)
top_mergers = {}
top_mergers['mergers.'] = mmergers.get_list(mfilter, 0)
top_mergers['mergers.last month'] = mmergers.get_list(mfilter, 31)
top_mergers['mergers.last year'] = mmergers.get_list(mfilter, 365)
top_participants = {}
top_participants['participants.'] = mparticipants.get_list(mfilter, 0)
top_participants['participants.last month'] = mparticipants.get_list(mfilter, 31)
top_participants['participants.last year'] = mparticipants.get_list(mfilter, 365)
top_core_reviewers = {}
top_core_reviewers['active_core_reviewers.'] = mcorereviewers.get_list(mfilter, 0)
top_core_reviewers['active_core_reviewers.last month'] = mcorereviewers.get_list(mfilter, 31)
top_core_reviewers['active_core_reviewers.last year'] = mcorereviewers.get_list(mfilter, 365)
# The order of the list item change so we can not check it
top_all = dict(top_reviewers.items() + top_openers.items() + top_mergers.items() + top_participants.items() + top_core_reviewers.items())
else:
filter_name = filter_.get_name()
if filter_name in ["company","domain","repository"]:
if filter_name in ["company","domain","repository"]:
top_all = {}
top_all['active_core_reviewers.'] = mcorereviewers.get_list(mfilter, 0)
top_all['active_core_reviewers.last month'] = mcorereviewers.get_list(mfilter, 31)
top_all['active_core_reviewers.last year'] = mcorereviewers.get_list(mfilter, 365)
else:
# Remove filters above if there are performance issues
top_all = mcorereviewers.get_list(mfilter)
else:
top_all = None
return (top_all)
def get_top_data(startdate, enddate, identities_db, filter_, npeople):
bots = Pullpo.get_bots()
top_all = None
mreviewers = DataSource.get_metrics("reviewers", Pullpo)
msubmitters = DataSource.get_metrics("submitters", Pullpo)
mclosers = DataSource.get_metrics("closers", Pullpo)
mmergers = DataSource.get_metrics("mergers", Pullpo)
mparticipants = DataSource.get_metrics("participants", Pullpo)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
if filter_ is None:
top_reviewers = {}
top_reviewers['reviewers'] = mreviewers.get_list(mfilter, 0)
top_reviewers['reviewers.last month'] = mreviewers.get_list(
mfilter, 31)
top_reviewers['reviewers.last year'] = mreviewers.get_list(
mfilter, 365)
top_submitters = {}
top_submitters['submitters.'] = msubmitters.get_list(mfilter, 0)
top_submitters['submitters.last month'] = msubmitters.get_list(
mfilter, 31)
top_submitters['submitters.last year'] = msubmitters.get_list(
mfilter, 365)
top_closers = {}
top_closers['closers.'] = mclosers.get_list(mfilter, 0)
top_closers['closers.last month'] = mclosers.get_list(mfilter, 31)
top_closers['closers.last year'] = mclosers.get_list(mfilter, 365)
top_mergers = {}
top_mergers['mergers.'] = mmergers.get_list(mfilter, 0)
top_mergers['mergers.last month'] = mmergers.get_list(mfilter, 31)
top_mergers['mergers.last year'] = mmergers.get_list(mfilter, 365)
top_participants = {}
top_participants['participants.'] = mparticipants.get_list(
mfilter, 0)
top_participants[
'participants.last month'] = mparticipants.get_list(
mfilter, 31)
top_participants[
'participants.last year'] = mparticipants.get_list(
mfilter, 365)
# The order of the list item change so we can not check it
top_all = dict(top_reviewers.items() + top_submitters.items() +
top_closers.items() + top_mergers.items() +
top_participants.items())
else:
logging.info("Pullpo does not support yet top for filters.")
return (top_all)
def ages_study_com(ds, items, period, startdate, enddate, destdir):
"""Perform ages study for companies, if it is specified in Report.
Produces JSON files for those studies.
Parameters
----------
ds: { SCM | ITS | MLS }
Data source
items: ??
Items
period: ??
Period
startdate: ??
Start date
enddate: ??
End date
destdir: string
Directory for writing the JSON files
"""
from vizgrimoire.report import Report
filter_name = "company"
studies = Report.get_studies()
ages = None
for study in studies:
if study.id == "ages":
ages = study
if ages is not None:
# Get config parameters for producing a connection
# to the database
config = Report.get_config()
db_identities = config['generic']['db_identities']
dbuser = config['generic']['db_user']
dbpass = config['generic']['db_password']
start_string = ds.get_name() + "_start_date"
end_string = ds.get_name() + "_end_date"
if start_string in config['r']:
startdate = "'" + config['r'][start_string] + "'"
if end_string in config['r']:
enddate = "'" + config['r'][end_string] + "'"
ds_dbname = ds.get_db_name()
dbname = config['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for item in items:
filter_item = Filter(filter_name, item)
metric_filters = MetricFilters(period, startdate, enddate,
filter_item.get_type_analysis())
obj = ages(dbcon, metric_filters)
res = obj.create_report(ds, destdir)
def get_top_data(startdate,
enddate,
identities_db,
filter_,
npeople,
threads_top=True):
msenders = DataSource.get_metrics("senders", MLS)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
top = {}
if filter_ is None:
top['senders.'] = msenders.get_list(mfilter, 0)
top['senders.last month'] = msenders.get_list(mfilter, 31)
top['senders.last year'] = msenders.get_list(mfilter, 365)
if threads_top:
top['threads.'] = MLS.getLongestThreads(
startdate, enddate, identities_db, npeople)
startdate = datetime.date.today() - datetime.timedelta(
days=365)
startdate = "'" + str(startdate) + "'"
top['threads.last year'] = MLS.getLongestThreads(
startdate, enddate, identities_db, npeople)
startdate = datetime.date.today() - datetime.timedelta(days=30)
startdate = "'" + str(startdate) + "'"
top['threads.last month'] = MLS.getLongestThreads(
startdate, enddate, identities_db, npeople)
else:
filter_name = filter_.get_name()
item = filter_.get_item()
if filter_name in [
"company", "domain", "repository", "domain", "country"
]:
if filter_name in [
"company", "domain", "repository", "domain", "country"
]:
top['senders.'] = msenders.get_list(mfilter, 0)
top['senders.last month'] = msenders.get_list(mfilter, 31)
top['senders.last year'] = msenders.get_list(mfilter, 365)
else:
# Remove filters above if there are performance issues
top = msenders.get_list(mfilter)
else:
top = None
return top
def __init__(self, dbcon = None, filters = None):
"""db connection and filter to be used"""
self.db = dbcon
self.filters = filters
if filters == None:
people_out = None
organizations_out = None
npeople = None
type_analysis = None
self.filters = MetricFilters(Metrics.default_period,
Metrics.default_start, Metrics.default_end,
type_analysis,
npeople, people_out, organizations_out)
def GetCommitsSummaryCompanies(period, startdate, enddate, identities_db,
num_organizations):
# This function returns the following dataframe structrure
# unixtime, date, week/month/..., company1, company2, ... company[num_organizations -1], others
# The 3 first fields are used for data and ordering purposes
# The "companyX" fields are those that provide info about that company
# The "Others" field is the aggregated value of the rest of the organizations
# Companies above num_organizations will be aggregated in Others
from vizgrimoire.SCM import SCM
metric = DataSource.get_metrics("organizations", SCM)
organizations = metric.get_list()
organizations = organizations['name']
first_organizations = {}
count = 1
for company in organizations:
company_name = "'" + company + "'"
type_analysis = ['company', company_name]
mcommits = DataSource.get_metrics("commits", SCM)
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
mfilter_orig = mcommits.filters
mcommits.filters = mfilter
commits = mcommits.get_ts()
mcommits.filters = mfilter_orig
# commits = EvolCommits(period, startdate, enddate, identities_db, ["company", company_name])
# commits = completePeriodIds(commits, period, startdate, enddate)
# Rename field commits to company name
commits[company] = commits["commits"]
del commits['commits']
if (count <= num_organizations):
#Case of organizations with entity in the dataset
first_organizations = dict(first_organizations.items() +
commits.items())
else:
#Case of organizations that are aggregated in the field Others
if 'Others' not in first_organizations:
first_organizations['Others'] = commits[company]
else:
first_organizations['Others'] = [
a + b for a, b in zip(first_organizations['Others'],
commits[company])
]
count = count + 1
#TODO: remove global variables...
first_organizations = completePeriodIds(first_organizations, period,
startdate, enddate)
return (first_organizations)
def get_trends(self, date, days):
#Specific needs for Added and Removed lines not considered in meta class Metrics
filters = self.filters
chardates = GetDates(date, days)
self.filters = MetricFilters(Metrics.default_period, chardates[1],
chardates[0], None)
last = self.get_agg()
if last['added_lines'] is None: last['added_lines'] = 0
last_added = int(last['added_lines'])
if last['removed_lines'] is None: last['removed_lines'] = 0
last_removed = int(last['removed_lines'])
self.filters = MetricFilters(Metrics.default_period, chardates[2],
chardates[1], None)
prev = self.get_agg()
if prev['added_lines'] is None: prev['added_lines'] = 0
prev_added = int(prev['added_lines'])
if prev['removed_lines'] is None: prev['removed_lines'] = 0
prev_removed = int(prev['removed_lines'])
data = {}
data['diff_netadded_lines_' + str(days)] = last_added - prev_added
data['percentage_added_lines_' + str(days)] = GetPercentageDiff(
prev_added, last_added)
data['diff_netremoved_lines_' +
str(days)] = last_removed - prev_removed
data['percentage_removed_lines_' + str(days)] = GetPercentageDiff(
prev_removed, last_removed)
data['added_lines_' + str(days)] = last_added
data['removed_lines_' + str(days)] = last_removed
#Returning filters to their original value
self.filters = filters
return (data)
def data_source_increment_activity(opts, people_out, affs_out):
# Per data source, the increment or decrement of the activity is displayed
dataset = {}
data_sources = ["Gits", "Tickets", "Mailing Lists", "Gerrit", "Askbot", "IRC"]
action = ["commits", "closed tickets", "sent emails", "submitted reviews", "posted questions", "messages"]
net_values = []
rel_values = [] #percentage wrt the previous 365 days
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities)
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities)
mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats, opts.dbidentities)
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities)
qaforums_dbcon = QAForumsQuery(opts.dbuser, opts.dbpassword, opts.dbqaforums, opts.dbidentities)
irc_dbcon = IRCQuery(opts.dbuser, opts.dbpassword, opts.dbirc, opts.dbidentities)
period = "month"
type_analysis = None
releases = opts.releases.split(",")[-2:]
startdate = "'"+releases[0]+"'"
enddate = "'"+releases[1]+"'"
filters = MetricFilters(period, startdate, enddate, None, 10, people_out, affs_out)
commits = scm.Commits(scm_dbcon, filters)
closed = its.Closed(its_dbcon, filters)
emails = mls.EmailsSent(mls_dbcon, filters)
submitted = scr.Submitted(scr_dbcon, filters)
questions = qa.Questions(qaforums_dbcon, filters)
messages = irc.Sent(irc_dbcon, filters)
from vizgrimoire.ITS import ITS
ITS.set_backend("jira")
net_values.append(commits.get_trends(releases[1], 90)["commits_90"])
rel_values.append(commits.get_trends(releases[1], 90)["percentage_commits_90"])
net_values.append(closed.get_trends(releases[1], 90)["closed_90"])
rel_values.append(closed.get_trends(releases[1], 90)["percentage_closed_90"])
net_values.append(emails.get_trends(releases[1], 90)["sent_90"])
rel_values.append(emails.get_trends(releases[1], 90)["percentage_sent_90"])
net_values.append(submitted.get_trends(releases[1], 90)["submitted_90"])
rel_values.append(submitted.get_trends(releases[1], 90)["percentage_submitted_90"])
net_values.append(questions.get_trends(releases[1], 90)["qsent_90"])
rel_values.append(questions.get_trends(releases[1], 90)["percentage_qsent_90"])
net_values.append(messages.get_trends(releases[1], 90)["sent_90"])
rel_values.append(messages.get_trends(releases[1], 90)["percentage_sent_90"])
createCSV({"datasource":data_sources, "metricsnames":action, "relativevalues":rel_values, "netvalues":net_values}, "./release/data_source_evolution.csv")
def _get_data(period, startdate, enddate, i_db, filter_, evol):
data = {}
type_analysis = None
if (filter_ is not None):
type_analysis = [filter_.get_name(), filter_.get_item()]
logging.warn(DownloadsDS.get_name() + " does not support filters.")
return data
if (evol):
metrics_on = DownloadsDS.get_metrics_core_ts()
else:
metrics_on = DownloadsDS.get_metrics_core_agg()
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
all_metrics = DownloadsDS.get_metrics_set(DownloadsDS)
for item in all_metrics:
if item.id not in metrics_on: continue
item.filters = mfilter
if evol is False:
mvalue = item.get_agg()
else:
mvalue = item.get_ts()
data = dict(data.items() + mvalue.items())
if evol is False:
init_date = DownloadsDS.get_date_init(startdate, enddate, None,
type_analysis)
end_date = DownloadsDS.get_date_end(startdate, enddate, None,
type_analysis)
data = dict(data.items() + init_date.items() +
end_date.items())
# Tendencies
metrics_trends = DownloadsDS.get_metrics_core_trends()
for i in [7, 30, 365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
period_data = item.get_trends(enddate, i)
data = dict(data.items() + period_data.items())
if filter_ is not None: studies = {}
else:
studies = DataSource.get_studies_data(DownloadsDS, period,
startdate, enddate, evol)
return dict(data.items() + studies.items())
def get_list(self):
# Just get commits per project
startdate = self.filters.startdate
enddate = self.filters.enddate
type_analysis = ['project', None]
period = None
evol = False
mcommits = Commits(self.db, self.filters)
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
mfilter_orig = mcommits.filters
mcommits.filters = mfilter
commits = mcommits.get_agg()
mcommits.filters = mfilter_orig
checkListArray(commits)
return commits
def get_top_data(startdate, enddate, identities_db, filter_, npeople):
top = {}
mauthors = DataSource.get_metrics("pulls", DockerHubDS)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
if filter_ is None:
top['pulls.'] = mauthors.get_list(mfilter, 0)
else:
logging.info("DockerHubDS does not support yet top for filters.")
return (top)
def get_list(self):
# Just get closed per project
startdate = self.filters.startdate
enddate = self.filters.enddate
type_analysis = ['project', None]
period = None
evol = False
mclosed = Closed(self.db, self.filters)
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
mfilter_orig = mclosed.filters
mclosed.filters = mfilter
closed = mclosed.get_agg()
mclosed.filters = mfilter_orig
checkListArray(closed)
return closed
def get_list(self):
# Just get sent per project
startdate = self.filters.startdate
enddate = self.filters.enddate
type_analysis = ['project', None]
period = None
evol = False
msent = EmailsSent(self.db, self.filters)
mfilter = MetricFilters(period, startdate, enddate, type_analysis)
mfilter_orig = msent.filters
msent.filters = mfilter
sent = msent.get_agg()
msent.filters = mfilter_orig
checkListArray(sent)
return sent
def test_create_reports_studies(self):
opts = read_options()
period = getPeriod(opts.granularity)
destdir = os.path.join("data","json")
from vizgrimoire.metrics.metrics_filter import MetricFilters
db_identities= Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
studies = Report.get_studies()
metric_filters = MetricFilters(period, startdate, enddate, [])
for ds in Report.get_data_sources():
ds_dbname = ds.get_db_name()
dbname = Report.get_config()['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for study in studies:
# logging.info("Creating report for " + study.id + " for " + ds.get_name())
try:
obj = study(dbcon, metric_filters)
obj.create_report(ds, destdir)
files = obj.get_report_files(ds)
if len(files) > 0:
for file in files:
f_test_json = os.path.join("json", file)
f_report_json = os.path.join(destdir, file)
if obj.get_definition()['id'] == "contributors_new_gone":
# authors is a dict with revtime which changes every execution
pass
else:
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
data = obj.result(ds)
if data is None: continue
test_json = os.path.join("json",ds.get_name()+"_"+obj.get_definition()['id']+".json")
if obj.get_definition()['id'] == "top_issues":
# Include time field which changes every execution
continue
else: self.assertTrue(self._compare_data(data, test_json))
except TypeError:
traceback.print_exc(file=sys.stdout)
logging.info(study.id + " does no support complete standard API. Not used when no available.")
continue
def get_top_data (startdate, enddate, identities_db, filter_, npeople):
top = {}
mauthors = DataSource.get_metrics("authors", Mediawiki)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
if filter_ is None:
top['authors.'] = mauthors.get_list(mfilter, 0)
top['authors.last month'] = mauthors.get_list(mfilter, 31)
top['authors.last year'] = mauthors.get_list(mfilter, 365)
else:
logging.info("Mediawiki does not support yet top for filters.")
return(top)
def get_top_data(startdate,
enddate,
identities_db,
filter_=None,
npeople=None):
def filter_ips(ips):
new_ips = {}
new_ips['downloads'] = ips['downloads']
new_ips['ips'] = []
for ip in ips['ips']:
# ipv4
new_ip_aux = ip.split(".")
new_ip = ip
if len(new_ip_aux) == 4:
new_ip = "x.x." + new_ip_aux[2] + "." + new_ip_aux[3]
# ipv6
new_ip_aux = ip.split(":")
if len(new_ip_aux) > 1:
new_ip = new_ip_aux[0] + ":X"
new_ips['ips'].append(new_ip)
return new_ips
top = {}
mips = DataSource.get_metrics("ips", DownloadsDS)
mpackages = DataSource.get_metrics("packages", DownloadsDS)
mpages = DataSource.get_metrics("pages", DownloadsDS)
mcountries = DataSource.get_metrics("countries", DownloadsDS)
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
if filter_ is None:
top['ips.'] = filter_ips(mips.get_list(mfilter, 0))
top['packages.'] = mpackages.get_list(mfilter, 0)
top['pages.'] = mpages.get_list()
top['countries.'] = mcountries.get_list()
else:
logging.info("DownloadsDS does not support yet top for filters.")
return top
def get_top_data_organizations(startdate, enddate, i_db, filter_, npeople):
top = {}
morganizations = DataSource.get_metrics("organizations", SCM)
if morganizations is None: return top
period = None
if filter_ is not None:
if filter_.get_name() == "project":
# TODO missing organizations_out
organizations_out = morganizations.filters.organizations_out
people_out = None
mfilter = MetricFilters(period, startdate, enddate,
filter_.get_type_analysis(), npeople,
people_out, organizations_out)
mfilter_orig = morganizations.filters
morganizations.filters = mfilter
top = morganizations.get_list()
morganizations.filters = mfilter_orig
return top
def projects_efficiency(opts, people_out, affs_out):
# BMI and time to review in mean per general project
scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities, opts.dbprojects)
scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities, opts.dbprojects)
its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities, opts.dbprojects)
bmi_list = []
time2review_list = []
bmi_its = []
period = "month"
releases = opts.releases.split(",")[-2:]
startdate = "'"+releases[0]+"'"
enddate = "'"+releases[1]+"'"
type_analysis = None
project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10,
people_out, affs_out)
scr_bmi = scr.BMISCR(scr_dbcon, project_filters)
time2review = scr.TimeToReview(scr_dbcon, project_filters)
# ITS BMI index
from vizgrimoire.ITS import ITS
ITS.set_backend("jira")
opened = its.Opened(its_dbcon, project_filters)
closed = its.Closed(its_dbcon, project_filters)
tickets_opened = opened.get_agg()["opened"]
tickets_closed = closed.get_agg()["closed"]
its_bmi = 0
if tickets_closed > 0:
its_bmi = round(float(tickets_closed)/float(tickets_opened), 2)
bmi_list.append(round(scr_bmi.get_agg()["bmiscr"], 2))
time2review_list.append(round(time2review.get_agg()["review_time_days_median"], 2))
bmi_its.append(its_bmi)
createCSV({"projects":"general", "bmi":bmi_list, "timereview":time2review_list, "bmiits":bmi_its}, "./release/integrated_projects_efficiency.csv")
def get_top_data(startdate, enddate, identities_db, filter_, npeople):
top = {}
attendees = DataSource.get_metrics("rsvps", EventsDS)
period = attendees.filters.period
type_analysis = None
mfilter = attendees.filters # updated filters
filters = None # original filters
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
if filter_ is not None:
filters = attendees.filters
attendees.filters = mfilter
top['rsvps.'] = attendees.get_list(mfilter, 0)
top['rsvps.last month'] = attendees.get_list(mfilter, 31)
top['rsvps.last year'] = attendees.get_list(mfilter, 365)
if filter_ is not None:
attendees.filters = filters
events = DataSource.get_metrics("events", EventsDS)
if filter_ is not None:
filters = events.filters
events.filters = mfilter
top['events.'] = events.get_list(mfilter, 0)
top['events.last month'] = events.get_list(mfilter, 31)
top['events.last year'] = events.get_list(mfilter, 365)
if filter_ is not None:
events.filters = filters
if filter_ is not None:
groups = DataSource.get_metrics("groups", EventsDS)
filters = groups.filters
groups.filters = mfilter
if filter_.get_name() <> 'repository':
top['groups.'] = groups.get_list(mfilter, 0)
top['groups.last month'] = groups.get_list(mfilter, 31)
top['groups.last year'] = groups.get_list(mfilter, 365)
groups.filters = filters
return top
def GetClosedSummaryCompanies(period, startdate, enddate, identities_db,
closed_condition, num_organizations):
from vizgrimoire.ITS import ITS
count = 1
first_organizations = {}
metric = DataSource.get_metrics("organizations", ITS)
organizations = metric.get_list()
organizations = organizations['name']
for company in organizations:
type_analysis = ["company", "'" + company + "'"]
filter_com = MetricFilters(period, startdate, enddate, type_analysis)
mclosed = ITS.get_metrics("closed", ITS)
mclosed.filters = filter_com
closed = mclosed.get_ts()
# Rename field closed to company name
closed[company] = closed["closed"]
del closed['closed']
if (count <= num_organizations):
#Case of organizations with entity in the dataset
first_organizations = dict(first_organizations.items() +
closed.items())
else:
#Case of organizations that are aggregated in the field Others
if 'Others' not in first_organizations:
first_organizations['Others'] = closed[company]
else:
first_organizations['Others'] = [
a + b for a, b in zip(first_organizations['Others'],
closed[company])
]
count = count + 1
first_organizations = completePeriodIds(first_organizations, period,
startdate, enddate)
return (first_organizations)
def get_top_data (cls, startdate, enddate, identities_db, filter_, npeople):
bots = cls.get_bots()
closed_condition = cls._get_closed_condition()
# TODO: It should be configurable from Automator
top_issues_on = False
top = None
mopeners = DataSource.get_metrics("openers", cls)
mclosers = DataSource.get_metrics("closers", cls)
# We should check this metric is ON
stories_openers = DataSource.get_metrics("stories_openers", cls)
if mopeners is None or mclosers is None: return None
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
if mclosers.filters.closed_condition is not None:
mfilter.closed_condition = mclosers.filters.closed_condition
if filter_ is None:
top_closers_data = {}
top_closers_data['closers.'] = mclosers.get_list(mfilter, 0)
top_closers_data['closers.last month']= mclosers.get_list(mfilter, 31)
top_closers_data['closers.last year']= mclosers.get_list(mfilter, 365)
top_openers_data = {}
top_openers_data['openers.'] = mopeners.get_list(mfilter, 0)
top_openers_data['openers.last month'] = mopeners.get_list(mfilter, 31)
top_openers_data['openers.last year'] = mopeners.get_list(mfilter, 365)
top = dict(top_closers_data.items() + top_openers_data.items())
if top_issues_on:
from vizgrimoire.analysis.top_issues import TopIssues
from vizgrimoire.report import Report
db_identities= Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
dbname = Report.get_config()['generic'][cls.get_db_name()]
dbcon = ITSQuery(dbuser, dbpass, dbname, db_identities)
metric_filters = MetricFilters(None, startdate, enddate, [])
top_issues_data = TopIssues(dbcon, metric_filters).result(cls)
top = dict(top.items() + top_issues_data.items())
if False and stories_openers is not None:
top_sopeners_data = {}
top_sopeners_data['stories_openers.'] = stories_openers.get_list(mfilter, 0)
top_sopeners_data['stories_openers.last month'] = stories_openers.get_list(mfilter, 31)
top_sopeners_data['stories_openers.last year'] = stories_openers.get_list(mfilter, 365)
top = dict(top.items() + top_sopeners_data.items())
else:
filter_name = filter_.get_name()
if filter_name in ["company","domain","repository"]:
if filter_name in ["company","domain","repository"]:
top = {}
top['closers.'] = mclosers.get_list(mfilter, 0)
top['closers.last month']= mclosers.get_list(mfilter, 31)
top['closers.last year']= mclosers.get_list(mfilter, 365)
else:
# Remove filters above if there are performance issues
top = mclosers.get_list(mfilter)
else:
top = None
return top
def __get_data__ (period, startdate, enddate, identities_db, filter_ = None, evol = False):
data = {}
DS = SCR
from vizgrimoire.GrimoireUtils import fill_and_order_items
type_analysis = None
if filter_ is not None:
type_analysis = [filter_.get_name(), filter_.get_item()]
from vizgrimoire.report import Report
automator = Report.get_config()
if evol:
metrics_on = DS.get_metrics_core_ts()
automator_metrics = DS.get_name()+"_metrics_ts"
else:
metrics_on = DS.get_metrics_core_agg()
automator_metrics = DS.get_name()+"_metrics_agg"
if automator_metrics in automator['r']:
metrics_on = automator['r'][automator_metrics].split(",")
logging.info(automator_metrics + " found ")
# print(metrics_on)
people_out = []
if "people_out" in Report.get_config()['r']:
people_out = Report.get_config()['r']["people_out"]
people_out = people_out.split(",")
metrics_reports = SCR.get_metrics_core_reports()
if filter_ is None:
from vizgrimoire.report import Report
reports_on = Report.get_config()['r']['reports'].split(",")
for r in metrics_reports:
if r in reports_on: metrics_on += [r]
if type_analysis and type_analysis[1] is None:
items = DS.get_filter_items(filter_, startdate, enddate, identities_db)
items = items.pop('name')
if DS.get_name()+"_start_date" in Report.get_config()['r']:
startdate = "'"+Report.get_config()['r'][DS.get_name()+"_start_date"]+"'"
if DS.get_name()+"_end_date" in Report.get_config()['r']:
enddate = "'"+Report.get_config()['r'][DS.get_name()+"_end_date"]+"'"
mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None)
all_metrics = SCR.get_metrics_set(SCR)
# SCR specific: remove some metrics from filters
if filter_ is not None:
metrics_not_filters = SCR.get_metrics_not_filters()
metrics_on_filters = list(set(metrics_on) - set(metrics_not_filters))
if filter_.get_name() == "repository":
if 'review_time' in metrics_on: metrics_on_filters+= ['review_time']
if 'submitted' in metrics_on: metrics_on_filters+= ['submitted']
metrics_on = metrics_on_filters
# END SCR specific
for item in all_metrics:
if item.id not in metrics_on: continue
# logging.info(item.id)
mfilter_orig = item.filters
mfilter.global_filter = mfilter_orig.global_filter
item.filters = mfilter
if not evol: mvalue = item.get_agg()
else: mvalue = item.get_ts()
if type_analysis and type_analysis[1] is None:
logging.info(item.id)
id_field = SCRQuery.get_group_field_alias(type_analysis[0])
mvalue = check_array_values(mvalue)
mvalue = fill_and_order_items(items, mvalue, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
# SCR SPECIFIC #
if evol:
if type_analysis and type_analysis[1] is None: pass
else:
metrics_on_changes = ['merged','abandoned','new']
for item in all_metrics:
if item.id in metrics_on_changes and filter_ is None:
mfilter_orig = item.filters
item.filters = mfilter
mvalue = item.get_ts_changes()
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
# END SCR SPECIFIC #
if not evol:
init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis)
end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis)
if type_analysis and type_analysis[1] is None:
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
init_date = fill_and_order_items(items, init_date, id_field,
evol, period, startdate, enddate)
end_date = fill_and_order_items(items, end_date, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + init_date.items() + end_date.items())
# Tendencies
metrics_trends = SCR.get_metrics_core_trends()
automator_metrics = DS.get_name()+"_metrics_trends"
if automator_metrics in automator['r']:
metrics_trends = automator['r'][automator_metrics].split(",")
for i in [7,30,365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
mfilter_orig = item.filters
item.filters = mfilter
period_data = item.get_trends(enddate, i)
item.filters = mfilter_orig
data = dict(data.items() + period_data.items())
if type_analysis and type_analysis[1] is None:
id_field = SCRQuery.get_group_field_alias(type_analysis[0])
period_data = fill_and_order_items(items, period_data, id_field)
data = dict(data.items() + period_data.items())
if filter_ is not None: studies_data = {}
else:
studies_data = DataSource.get_studies_data(SCR, period, startdate, enddate, evol)
data = dict(data.items() + studies_data.items())
return data
def get_list (self):
#TODO: warning -> not using GetSQLReportFrom/Where
q = "SELECT re.url as name, COUNT(DISTINCT(pr.id)) AS submitted "+\
" FROM pull_requests pr, repositories re "+\
" WHERE pr.repo_id = re.id AND "+\
" pr.created_at >="+ self.filters.startdate+ " AND "+\
" pr.created_at < "+ self.filters.enddate +\
" GROUP BY re.id "+\
" ORDER BY submitted DESC, name "
names = self.db.ExecuteQuery(q)
if not isinstance(names['name'], (list)): names['name'] = [names['name']]
return(names)
if __name__ == '__main__':
filters = MetricFilters("month", "'2014-01-01'", "'2015-01-01'")
dbcon = PullpoQuery("root", "", "xxxxx", "xxxxx")
timeto = TimeToClose(dbcon, filters)
print timeto.get_agg()
print timeto.get_ts()
timeto = TimeToMerge(dbcon, filters)
print timeto.get_agg()
print timeto.get_ts()
filters1 = filters.copy()
type_analysis = ["company", "'xxxxx'"]
filters1.type_analysis = type_analysis
company = TimeToMerge(dbcon, filters1)
print company.get_ts()
def get_metrics_data(DS, period, startdate, enddate, identities_db,
filter_ = None, evol = False):
""" Get basic data from all core metrics """
from vizgrimoire.GrimoireUtils import fill_and_order_items
from vizgrimoire.ITS import ITS
from vizgrimoire.MLS import MLS
data = {}
dsquery = DSQuery
if DS == ITS: dsquery = ITSQuery
if DS == MLS: dsquery = MLSQuery
from vizgrimoire.report import Report
automator = Report.get_config()
if evol:
metrics_on = DS.get_metrics_core_ts()
automator_metrics = DS.get_name()+"_metrics_ts"
else:
metrics_on = DS.get_metrics_core_agg()
automator_metrics = DS.get_name()+"_metrics_agg"
if automator_metrics in automator['r']:
metrics_on = automator['r'][automator_metrics].split(",")
people_out = []
if "people_out" in Report.get_config()['r']:
people_out = Report.get_config()['r']["people_out"]
people_out = people_out.split(",")
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
if type_analysis and type_analysis[1] is None:
# We need the items for filling later values in group by queries
items = DS.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: return data
items = items.pop('name')
if DS.get_name()+"_startdate" in Report.get_config()['r']:
startdate = Report.get_config()['r'][DS.get_name()+"_startdate"]
if DS.get_name()+"_enddate" in Report.get_config()['r']:
enddate = Report.get_config()['r'][DS.get_name()+"_enddate"]
# TODO: the hardcoded 10 should be removed, and use instead the npeople provided
# in the config file.
mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None)
metrics_reports = DS.get_metrics_core_reports()
all_metrics = DS.get_metrics_set(DS)
# Reports = filters metrics not available inside filters
if type_analysis is None:
from vizgrimoire.report import Report
reports_on = Report.get_config()['r']['reports'].split(",")
for r in metrics_reports:
if r in reports_on: metrics_on += [r]
for item in all_metrics:
# print item
if item.id not in metrics_on: continue
mfilter_orig = item.filters
mfilter.global_filter = mfilter_orig.global_filter
mfilter.set_closed_condition(mfilter_orig.closed_condition)
item.filters = mfilter
if evol: mvalue = item.get_ts()
else: mvalue = item.get_agg()
if type_analysis and type_analysis[1] is None and mvalue:
logging.info(item.id)
id_field = None
# Support for combined filters
for idf in mvalue.keys():
if "CONCAT(" in idf:
id_field = idf
break
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
mvalue = fill_and_order_items(items, mvalue, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
if not evol:
init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis)
end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis)
if type_analysis and type_analysis[1] is None:
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
init_date = fill_and_order_items(items, init_date, id_field,
evol, period, startdate, enddate)
end_date = fill_and_order_items(items, end_date, id_field,
evol, period, startdate, enddate)
if init_date is None: init_date = {}
if end_date is None: end_date = {}
data = dict(data.items() + init_date.items() + end_date.items())
# Tendencies
metrics_trends = DS.get_metrics_core_trends()
automator_metrics = DS.get_name()+"_metrics_trends"
if automator_metrics in automator['r']:
metrics_trends = automator['r'][automator_metrics].split(",")
for i in [7,30,365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
mfilter_orig = item.filters
item.filters = mfilter
period_data = item.get_trends(enddate, i)
item.filters = mfilter_orig
if type_analysis and type_analysis[1] is None:
group_field = dsquery.get_group_field_alias(type_analysis[0])
period_data = fill_and_order_items(items, period_data, group_field)
data = dict(data.items() + period_data.items())
return data