def get_filter_items(filter_, startdate, enddate, identities_db):
items = None
filter_name = filter_.get_name()
if (filter_name == "repository"):
metric = DataSource.get_metrics("repositories", SCR)
elif (filter_name == "company"):
metric = DataSource.get_metrics("organizations", SCR)
elif (filter_name == "country"):
metric = DataSource.get_metrics("countries", SCR)
elif (filter_name == "domain"):
metric = DataSource.get_metrics("domains", SCR)
elif (filter_name == "project"):
metric = DataSource.get_metrics("projects", SCR)
elif (filter_name == "people2"):
metric = DataSource.get_metrics("people2", SCR)
else:
logging.error("SCR " + filter_name + " not supported")
return items
# workaround due to bug https://phabricator.wikimedia.org/T116484
from vizgrimoire.report import Report
automator = Report.get_config()
DS = SCR
if DS.get_name()+"_start_date" in Report.get_config()['r']:
metric.filters.startdate = "'"+Report.get_config()['r'][DS.get_name()+"_start_date"]+"'"
if DS.get_name()+"_end_date" in Report.get_config()['r']:
metric.filters.enddate = "'"+Report.get_config()['r'][DS.get_name()+"_end_date"]+"'"
# end
items = metric.get_list()
return items
def ages_study_com (ds, items, period,
startdate, enddate, destdir):
"""Perform ages study for companies, if it is specified in Report.
Produces JSON files for those studies.
Parameters
----------
ds: { SCM | ITS | MLS }
Data source
items: ??
Items
period: ??
Period
startdate: ??
Start date
enddate: ??
End date
destdir: string
Directory for writing the JSON files
"""
from vizgrimoire.report import Report
filter_name = "company"
studies = Report.get_studies()
ages = None
for study in studies:
if study.id == "ages":
ages = study
if ages is not None:
# Get config parameters for producing a connection
# to the database
config = Report.get_config()
db_identities = config['generic']['db_identities']
dbuser = config['generic']['db_user']
dbpass = config['generic']['db_password']
start_string = ds.get_name() + "_start_date"
end_string = ds.get_name() + "_end_date"
if start_string in config['r']:
startdate = "'" + config['r'][start_string] + "'"
if end_string in config['r']:
enddate = "'" + config['r'][end_string] + "'"
ds_dbname = ds.get_db_name()
dbname = config['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for item in items :
filter_item = Filter(filter_name, item)
metric_filters = MetricFilters(
period, startdate, enddate,
filter_item.get_type_analysis()
)
obj = ages(dbcon, metric_filters)
res = obj.create_report(ds, destdir)
def ages_study_com(ds, items, period, startdate, enddate, destdir):
"""Perform ages study for companies, if it is specified in Report.
Produces JSON files for those studies.
Parameters
----------
ds: { SCM | ITS | MLS }
Data source
items: ??
Items
period: ??
Period
startdate: ??
Start date
enddate: ??
End date
destdir: string
Directory for writing the JSON files
"""
from vizgrimoire.report import Report
filter_name = "company"
studies = Report.get_studies()
ages = None
for study in studies:
if study.id == "ages":
ages = study
if ages is not None:
# Get config parameters for producing a connection
# to the database
config = Report.get_config()
db_identities = config['generic']['db_identities']
dbuser = config['generic']['db_user']
dbpass = config['generic']['db_password']
start_string = ds.get_name() + "_start_date"
end_string = ds.get_name() + "_end_date"
if start_string in config['r']:
startdate = "'" + config['r'][start_string] + "'"
if end_string in config['r']:
enddate = "'" + config['r'][end_string] + "'"
ds_dbname = ds.get_db_name()
dbname = config['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for item in items:
filter_item = Filter(filter_name, item)
metric_filters = MetricFilters(period, startdate, enddate,
filter_item.get_type_analysis())
obj = ages(dbcon, metric_filters)
res = obj.create_report(ds, destdir)
def get_filter_bots(filter_):
from vizgrimoire.report import Report
bots = []
# If not using Report (automator) bots are not supported.
if Report.get_config() == None:
return bots
if filter_.get_name_plural()+'_out' in Report.get_config()['r']:
fbots = Report.get_config()['r'][filter_.get_name_plural()+'_out']
bots = fbots.split(",")
logging.info("BOTS for " + filter_.get_name_plural())
logging.info(bots)
return bots
def get_filter_bots(filter_):
from vizgrimoire.report import Report
bots = []
# If not using Report (automator) bots are not supported.
if Report.get_config() == None:
return bots
if filter_.get_name_plural() + '_out' in Report.get_config()['r']:
fbots = Report.get_config()['r'][filter_.get_name_plural() +
'_out']
bots = fbots.split(",")
logging.info("BOTS for " + filter_.get_name_plural())
logging.info(bots)
return bots
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir, filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items:
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate,
identities_db, filter_item,
npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir,
filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items :
# item_name = "'"+ item+ "'"
logging.info (item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(IRC()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = QAForums.get_filter_items(filter_, startdate, enddate, identities_db)
if items == None:
return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(QAForums()))
createJSON(file_items, fn)
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = QAForums.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(QAForums()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = QAForums.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(QAForums()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
items_files = [item.replace('/', '_').replace("<","__").replace(">","___")
for item in items]
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_files, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'sent_365' : [], 'senders_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_.get_name(), item)
evol_data = MLS.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(MLS()))
createJSON(evol_data, fn)
agg = MLS.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(MLS()))
createJSON(agg, fn)
if filter_name in ("domain", "company", "repository"):
items_list['name'].append(item.replace('/', '_').replace("<","__").replace(">","___"))
items_list['sent_365'].append(agg['sent_365'])
items_list['senders_365'].append(agg['senders_365'])
top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False)
createJSON(top_senders, destdir+"/"+filter_item.get_top_filename(MLS()))
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = MLS
if False:
summary = MLS.get_filter_summary(
filter_, period, startdate, enddate,
identities_db, 10, self.db.projects_db
)
createJSON (summary,
destdir + "/" + filter_.get_summary_filename(MLS))
# Perform ages study, if it is specified in Report
MLS.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items:
item = item.replace("'", "\\'")
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db,
filter_item, npeople, False)
createJSON(top_senders,
destdir + "/" + filter_item.get_top_filename(MLS()))
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("company","project","repository"):
top_mergers = SCR.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(SCR()))
createJSON(top_mergers, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items:
# item_name = "'"+ item+ "'"
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(IRC()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = ReleasesDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = cls.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
createJSON(evol_data, fn)
agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(agg, fn)
if filter_name in ["domain", "company", "repository"]:
items_list['name'].append(item.replace('/', '_'))
items_list['closed_365'].append(agg['closed_365'])
items_list['closers_365'].append(agg['closers_365'])
if filter_name in ["company","domain","repository"]:
top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(cls()))
createJSON(top, fn)
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = ITS
summary = cls.get_filter_summary(
filter_, period, startdate, enddate,
identities_db, 10
)
createJSON (summary,
destdir + "/" + filter_.get_summary_filename(cls))
# Perform ages study, if it is specified in Report
cls.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = cls.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
createJSON(evol_data, fn)
agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(agg, fn)
if filter_name in ["domain", "company", "repository"]:
items_list['name'].append(item.replace('/', '_'))
items_list['closed_365'].append(agg['closed_365'])
items_list['closers_365'].append(agg['closers_365'])
if filter_name in ["company","domain","repository"]:
top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(cls()))
createJSON(top, fn)
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = ITS
#summary = cls.get_filter_summary(
# filter_, period, startdate, enddate,
# identities_db, 10
# )
#createJSON (summary,
# destdir + "/" + filter_.get_summary_filename(cls))
# Perform ages study, if it is specified in Report
cls.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
# Include metrics to sort in javascript.
items_list = {
"name": [],
"review_time_days_median": [],
"submitted": []
}
for item in items:
item_file = item.replace("/", "_")
items_list["name"].append(item_file)
logging.info(item)
filter_item = Filter(filter_name, item)
evol = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(SCR()))
createJSON(evol, fn)
# Static
agg = SCR.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCR()))
createJSON(agg, fn)
if 'submitted' in agg:
items_list["submitted"].append(agg["submitted"])
else:
items_list["submitted"].append("NA")
if 'review_time_days_median' in agg:
items_list["review_time_days_median"].append(
agg['review_time_days_median'])
else:
items_list["review_time_days_median"].append("NA")
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items_list, fn)
SCR.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_" + item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name': [], 'events_365': [], 'rsvps_365': []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate,
enddate, identities_db,
filter_item)
fn = os.path.join(
destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def get_its_from_organizations(self):
automator = Report.get_config()
identities_db = automator['generic']['db_identities']
from_ = """
FROM issues i
JOIN people_uidentities pup ON i.submitted_by = pup.people_id
JOIN %s.enrollments enr ON enr.uuid = pup.uuid
JOIN %s.organizations org ON org.id = enr.organization_id
""" % (identities_db, identities_db)
return from_
def get_its_from_organizations(self):
automator = Report.get_config()
identities_db = automator['generic']['db_identities']
from_ = """
FROM issues i
JOIN people_uidentities pup ON i.submitted_by = pup.people_id
JOIN %s.enrollments enr ON enr.uuid = pup.uuid
JOIN %s.organizations org ON org.id = enr.organization_id
""" % (identities_db, identities_db)
return from_
def get_mls_from_organizations(self):
automator = Report.get_config()
identities_db = automator['generic']['db_identities']
from_ = """
FROM messages m
JOIN messages_people mp ON m.message_ID = mp.message_id
JOIN people_uidentities pup ON mp.email_address = pup.people_id
JOIN %s.enrollments enr ON enr.upeople_id = pup.uuid
JOIN %s.organizations org ON org.id = enr.organization_id
""" % (identities_db, identities_db)
return from_
def get_mls_from_organizations(self):
automator = Report.get_config()
identities_db = automator['generic']['db_identities']
from_ = """
FROM messages m
JOIN messages_people mp ON m.message_ID = mp.message_id
JOIN people_uidentities pup ON mp.email_address = pup.people_id
JOIN %s.enrollments enr ON enr.upeople_id = pup.uuid
JOIN %s.organizations org ON org.id = enr.organization_id
""" % (identities_db, identities_db)
return from_
def get_top_data (startdate, enddate, i_db, filter_, npeople):
from vizgrimoire.report import Report
top = {}
data = SCM.get_top_data_authors (startdate, enddate, i_db, filter_, npeople)
top = dict(top.items() + data.items())
organizations_on = False
if Report.get_filter_automator('company') is not None:
organizations_on = True
if organizations_on:
data = SCM.get_top_data_organizations (startdate, enddate, i_db, filter_, npeople)
top = dict(top.items() + data.items())
return top
def get_studies_data(ds, period, startdate, enddate, evol):
""" Get data from studies to be included in agg and evol global JSONs """
from vizgrimoire.report import Report
data = {}
db_identities = Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
studies = Report.get_studies()
metric_filters = Report.get_default_filter()
ds_dbname = ds.get_db_name()
dbname = Report.get_config()['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
evol_txt = "evol"
if not evol: evol_txt = "agg"
logging.info("Creating studies for " + ds.get_name() + " " + evol_txt)
for study in studies:
try:
obj = study(dbcon, metric_filters)
if evol:
res = obj.get_ts(ds)
else:
res = obj.get_agg(ds)
if res is not None:
data = dict(res.items() + data.items())
except TypeError:
# logging.info(study.id + " does no support standard API. Not used.")
pass
return data
def get_studies_data(ds, period, startdate, enddate, evol):
""" Get data from studies to be included in agg and evol global JSONs """
from vizgrimoire.report import Report
data = {}
db_identities = Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
studies = Report.get_studies()
metric_filters = Report.get_default_filter()
ds_dbname = ds.get_db_name()
dbname = Report.get_config()['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
evol_txt = "evol"
if not evol: evol_txt = "agg"
logging.info("Creating studies for " + ds.get_name() + " " + evol_txt)
for study in studies:
try:
obj = study(dbcon, metric_filters)
if evol:
res = obj.get_ts(ds)
else:
res = obj.get_agg(ds)
if res is not None:
data = dict(res.items() + data.items())
except TypeError:
# logging.info(study.id + " does no support standard API. Not used.")
pass
return data
def get_scm_from_organizations(self, committers = False):
automator = Report.get_config()
identities_db = automator['generic']['db_identities']
if (committers): field = "s.committer_id"
else: field = "s.author_id"
from_ = """
FROM scmlog s
JOIN actions a ON a.commit_id = s.id
JOIN people_uidentities pup ON %s = pup.people_id
JOIN %s.enrollments enr ON enr.uuid = pup.uuid
JOIN %s.organizations org ON org.id = enr.organization_id
""" % (field, identities_db, identities_db)
return from_
def get_top_data(startdate, enddate, i_db, filter_, npeople):
from vizgrimoire.report import Report
top = {}
data = SCM.get_top_data_authors(startdate, enddate, i_db, filter_,
npeople)
top = dict(top.items() + data.items())
organizations_on = False
if Report.get_filter_automator('company') is not None:
organizations_on = True
if organizations_on:
data = SCM.get_top_data_organizations(startdate, enddate, i_db,
filter_, npeople)
top = dict(top.items() + data.items())
return top
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name' : [], 'events_365' : [], 'rsvps_365' : []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCM.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCM()))
escaped_items = [i.replace('/', '_') for i in items]
logging.info("D1")
createJSON(escaped_items, fn)
for item in items:
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_name, item)
evol_data = SCM.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(SCM()))
createJSON(evol_data, fn)
agg = SCM.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCM()))
createJSON(agg, fn)
SCM.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com(items, period, startdate, enddate, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
# Include metrics to sort in javascript.
items_list = {"name":[],"review_time_days_median":[],"submitted":[]}
for item in items :
item_file = item.replace("/","_")
items_list["name"].append(item_file)
logging.info (item)
filter_item = Filter(filter_name, item)
evol = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCR()))
createJSON(evol, fn)
# Static
agg = SCR.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCR()))
createJSON(agg, fn)
if 'submitted' in agg:
items_list["submitted"].append(agg["submitted"])
else: items_list["submitted"].append("NA")
if 'review_time_days_median' in agg:
items_list["review_time_days_median"].append(agg['review_time_days_median'])
else: items_list["review_time_days_median"].append("NA")
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items_list, fn)
SCR.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
def _get_metrics_for_pending_all(self, isevol):
""" Return the metric for all items normalized """
metrics = self._get_metrics_for_pending()
if isevol:
submitted = metrics['submitted'].get_ts()
merged = metrics['merged'].get_ts()
abandoned = metrics['abandoned'].get_ts()
else:
submitted = metrics['submitted'].get_agg()
merged = metrics['merged'].get_agg()
abandoned = metrics['abandoned'].get_agg()
from vizgrimoire.report import Report
filter = Report.get_filter(self.filters.type_analysis[0])
items = Pullpo.get_filter_items(filter, self.filters.startdate,
self.filters.enddate, self.db.identities_db)
items = items.pop('name')
from vizgrimoire.GrimoireUtils import fill_and_order_items
id_field = self.db.get_group_field_alias(self.filters.type_analysis[0])
submitted = check_array_values(submitted)
merged = check_array_values(merged)
abandoned = check_array_values(abandoned)
submitted = fill_and_order_items(items, submitted, id_field,
isevol, self.filters.period,
self.filters.startdate, self.filters.enddate)
merged = fill_and_order_items(items, merged, id_field,
isevol, self.filters.period,
self.filters.startdate, self.filters.enddate)
abandoned = fill_and_order_items(items, abandoned, id_field,
isevol, self.filters.period,
self.filters.startdate, self.filters.enddate)
metrics_for_pendig_all = {
id_field: submitted[id_field],
"submitted": submitted["submitted"],
"merged": merged["merged"],
"abandoned": abandoned["abandoned"]
}
if isevol:
metrics_for_pendig_all[self.filters.period] = submitted[self.filters.period]
return metrics_for_pendig_all
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items :
item = item.replace("'", "\\'")
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False)
createJSON(top_senders, destdir+"/"+filter_item.get_top_filename(MLS()))
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = QAForums.get_filter_items(filter_, startdate, enddate,
identities_db)
if items == None:
return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_" + item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(QAForums()))
createJSON(file_items, fn)
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = QAForums.get_evolutionary_data(period, startdate,
enddate, identities_db,
filter_item)
fn = os.path.join(
destdir, filter_item.get_evolutionary_filename(QAForums()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = QAForums.get_agg_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_static_filename(QAForums()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCM.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCM()))
escaped_items = [i.replace('/','_') for i in items]
logging.info("D1")
createJSON(escaped_items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = SCM.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCM()))
createJSON(evol_data, fn)
agg = SCM.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCM()))
createJSON(agg, fn)
SCM.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com (items, period, startdate, enddate, destdir)
def get_top_data (cls, startdate, enddate, identities_db, filter_, npeople):
bots = cls.get_bots()
closed_condition = cls._get_closed_condition()
# TODO: It should be configurable from Automator
top_issues_on = False
top = None
mopeners = DataSource.get_metrics("openers", cls)
mclosers = DataSource.get_metrics("closers", cls)
# We should check this metric is ON
stories_openers = DataSource.get_metrics("stories_openers", cls)
if mopeners is None or mclosers is None: return None
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople)
if mclosers.filters.closed_condition is not None:
mfilter.closed_condition = mclosers.filters.closed_condition
if filter_ is None:
top_closers_data = {}
top_closers_data['closers.'] = mclosers.get_list(mfilter, 0)
top_closers_data['closers.last month']= mclosers.get_list(mfilter, 31)
top_closers_data['closers.last year']= mclosers.get_list(mfilter, 365)
top_openers_data = {}
top_openers_data['openers.'] = mopeners.get_list(mfilter, 0)
top_openers_data['openers.last month'] = mopeners.get_list(mfilter, 31)
top_openers_data['openers.last year'] = mopeners.get_list(mfilter, 365)
top = dict(top_closers_data.items() + top_openers_data.items())
if top_issues_on:
from vizgrimoire.analysis.top_issues import TopIssues
from vizgrimoire.report import Report
db_identities= Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
dbname = Report.get_config()['generic'][cls.get_db_name()]
dbcon = ITSQuery(dbuser, dbpass, dbname, db_identities)
metric_filters = MetricFilters(None, startdate, enddate, [])
top_issues_data = TopIssues(dbcon, metric_filters).result(cls)
top = dict(top.items() + top_issues_data.items())
if False and stories_openers is not None:
top_sopeners_data = {}
top_sopeners_data['stories_openers.'] = stories_openers.get_list(mfilter, 0)
top_sopeners_data['stories_openers.last month'] = stories_openers.get_list(mfilter, 31)
top_sopeners_data['stories_openers.last year'] = stories_openers.get_list(mfilter, 365)
top = dict(top.items() + top_sopeners_data.items())
else:
filter_name = filter_.get_name()
if filter_name in ["company","domain","repository"]:
if filter_name in ["company","domain","repository"]:
top = {}
top['closers.'] = mclosers.get_list(mfilter, 0)
top['closers.last month']= mclosers.get_list(mfilter, 31)
top['closers.last year']= mclosers.get_list(mfilter, 365)
else:
# Remove filters above if there are performance issues
top = mclosers.get_list(mfilter)
else:
top = None
return top
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = Mediawiki.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
items_files = [
item.replace('/', '_').replace("<", "__").replace(">", "___")
for item in items
]
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_files, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name': [], 'sent_365': [], 'senders_365': []}
else:
items_list = items
for item in items:
item = item.replace("'", "\\'")
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_.get_name(), item)
evol_data = MLS.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(MLS()))
createJSON(evol_data, fn)
agg = MLS.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(MLS()))
createJSON(agg, fn)
if filter_name in ("domain", "company", "repository"):
items_list['name'].append(
item.replace('/', '_').replace("<",
"__").replace(">", "___"))
items_list['sent_365'].append(agg['sent_365'])
items_list['senders_365'].append(agg['senders_365'])
top_senders = MLS.get_top_data(startdate, enddate, identities_db,
filter_item, npeople, False)
createJSON(top_senders,
destdir + "/" + filter_item.get_top_filename(MLS()))
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = MLS
if False:
summary = MLS.get_filter_summary(filter_, period, startdate,
enddate, identities_db, 10,
self.db.projects_db)
createJSON(summary,
destdir + "/" + filter_.get_summary_filename(MLS))
# Perform ages study, if it is specified in Report
MLS.ages_study_com(items, period, startdate, enddate, destdir)
# env vars for R
os.environ["LANG"] = ""
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.metrics.metric import Metric
from vizgrimoire.SCM import SCM
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
logging.info("Grimoire Metrics Tool")
opts = get_options()
Report.init(opts.automator_file, opts.metrics_path)
# metrics = SCM.get_metrics_definition()
# for name in metrics:
# print name
dss = Report.get_data_sources()
if (opts.data_source):
ds = Report.get_data_source(opts.data_source)
dss = [ds]
if ds is None:
logging.error("Data source not found " + opts.data_source)
dss = []
total_metrics = 0
# env vars for R
os.environ["LANG"] = ""
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.metrics.metric import Metric
from vizgrimoire.SCM import SCM
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s')
logging.info("Grimoire Metrics Tool")
opts = get_options()
Report.init(opts.automator_file, opts.metrics_path)
# metrics = SCM.get_metrics_definition()
# for name in metrics:
# print name
dss = Report.get_data_sources()
if (opts.data_source):
ds = Report.get_data_source(opts.data_source)
dss = [ds]
if ds is None:
logging.error("Data source not found " + opts.data_source)
dss = []
total_metrics = 0
def result(self, data_source = None, destdir = None):
if data_source != SCM or destdir is None: return None
automator = Report.get_config()
db_identities = automator["generic"]["db_identities"]
dbuser = automator["generic"]["db_user"]
dbpass = automator["generic"]["db_password"]
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
start_year = int(start_date.split("-")[0])
end_year = int(end_date.split("-")[0])
activity = {}
activity['name'] = []
# Commits
data = self.db.ExecuteQuery(self.get_sql_commits())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("commits", activity, start_year, end_year)
# Authors
data = self.db.ExecuteQuery(self.get_sql_authors())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("authors", activity, start_year, end_year)
# Committers
data = self.db.ExecuteQuery(self.get_sql_committers())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("committers", activity, start_year, end_year)
# Committers active: only valid for today
data = self.db.ExecuteQuery(self.get_sql_committers(None, True))
data = self._convert_dict_field(data, "committers_active","committers-active")
activity = self.add_organizations_data (activity, data)
# Committers inactive: only valid for today
activity['committers-inactive'] = \
[ activity['committers'][i] - activity['committers-active'][i] \
for i in range(0, len(activity['committers']))]
activity['committers-percent-active'] = []
for i in range(0, len(activity['committers'])):
if activity['committers'][i] == 0:
activity['committers-percent-active'].append(100)
else:
activity['committers-percent-active'].append(\
(activity['committers-active'][i]*100) / activity['committers'][i])
# Actions
data = self.db.ExecuteQuery(self.get_sql_actions())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("actions", activity, start_year, end_year)
# Source lines of code added
data = self.db.ExecuteQuery(self.get_sql_lines_added())
data = self._convert_dict_field(data, "lines_added", "lines-added")
activity = self.add_organizations_data (activity, data)
self.add_metric_years("lines-added", activity, start_year, end_year)
# Source lines of code removed
data = self.db.ExecuteQuery(self.get_sql_lines_removed())
data = self._convert_dict_field(data, "lines_removed","lines-removed")
activity = self.add_organizations_data (activity, data)
self.add_metric_years("lines-removed", activity, start_year, end_year)
# Source lines of code total (added+removed)
data = self.db.ExecuteQuery(self.get_sql_lines_total())
data = self._convert_dict_field(data, "lines_total","lines-total")
activity = self.add_organizations_data (activity, data)
self.add_metric_years("lines-total", activity, start_year, end_year)
# Lines per commit
self.add_metric_lines_commit(activity, start_year, end_year)
# We need to change the db to tickets
dbname = automator["generic"]["db_bicho"]
dsquery = ITS.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
self.db = dbcon
# Tickets opened
data = self.db.ExecuteQuery(self.get_sql_opened())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("opened", activity, start_year, end_year)
# Tickets closed
data = self.db.ExecuteQuery(self.get_sql_closed())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("closed", activity, start_year, end_year)
# Messages sent
dbname = automator["generic"]["db_mlstats"]
dsquery = MLS.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
self.db = dbcon
data = self.db.ExecuteQuery(self.get_sql_sent())
activity = self.add_organizations_data (activity, data)
self.add_metric_years("sent", activity, start_year, end_year)
createJSON(activity, destdir+"/organizations-activity.json")
logging.info(destdir+"/organizations-activity.json created")
def result(self, data_source=None, destdir=None):
if data_source != SCM or destdir is None: return None
automator = Report.get_config()
db_identities = automator["generic"]["db_identities"]
dbuser = automator["generic"]["db_user"]
dbpass = automator["generic"]["db_password"]
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
start_year = int(start_date.split("-")[0])
end_year = int(end_date.split("-")[0])
activity = {}
activity['name'] = []
# Commits
data = self.db.ExecuteQuery(self.get_sql_commits())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("commits", activity, start_year, end_year)
# Authors
data = self.db.ExecuteQuery(self.get_sql_authors())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("authors", activity, start_year, end_year)
# Committers
data = self.db.ExecuteQuery(self.get_sql_committers())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("committers", activity, start_year, end_year)
# Committers active: only valid for today
data = self.db.ExecuteQuery(self.get_sql_committers(None, True))
data = self._convert_dict_field(data, "committers_active",
"committers-active")
activity = self.add_organizations_data(activity, data)
# Committers inactive: only valid for today
activity['committers-inactive'] = \
[ activity['committers'][i] - activity['committers-active'][i] \
for i in range(0, len(activity['committers']))]
activity['committers-percent-active'] = []
for i in range(0, len(activity['committers'])):
if activity['committers'][i] == 0:
activity['committers-percent-active'].append(100)
else:
activity['committers-percent-active'].append(\
(activity['committers-active'][i]*100) / activity['committers'][i])
# Actions
data = self.db.ExecuteQuery(self.get_sql_actions())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("actions", activity, start_year, end_year)
# Source lines of code added
data = self.db.ExecuteQuery(self.get_sql_lines_added())
data = self._convert_dict_field(data, "lines_added", "lines-added")
activity = self.add_organizations_data(activity, data)
self.add_metric_years("lines-added", activity, start_year, end_year)
# Source lines of code removed
data = self.db.ExecuteQuery(self.get_sql_lines_removed())
data = self._convert_dict_field(data, "lines_removed", "lines-removed")
activity = self.add_organizations_data(activity, data)
self.add_metric_years("lines-removed", activity, start_year, end_year)
# Source lines of code total (added+removed)
data = self.db.ExecuteQuery(self.get_sql_lines_total())
data = self._convert_dict_field(data, "lines_total", "lines-total")
activity = self.add_organizations_data(activity, data)
self.add_metric_years("lines-total", activity, start_year, end_year)
# Lines per commit
self.add_metric_lines_commit(activity, start_year, end_year)
# We need to change the db to tickets
dbname = automator["generic"]["db_bicho"]
dsquery = ITS.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
self.db = dbcon
# Tickets opened
data = self.db.ExecuteQuery(self.get_sql_opened())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("opened", activity, start_year, end_year)
# Tickets closed
data = self.db.ExecuteQuery(self.get_sql_closed())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("closed", activity, start_year, end_year)
# Messages sent
dbname = automator["generic"]["db_mlstats"]
dsquery = MLS.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
self.db = dbcon
data = self.db.ExecuteQuery(self.get_sql_sent())
activity = self.add_organizations_data(activity, data)
self.add_metric_years("sent", activity, start_year, end_year)
createJSON(activity, destdir + "/organizations-activity.json")
logging.info(destdir + "/organizations-activity.json created")
def __get_data__ (period, startdate, enddate, identities_db, filter_ = None, evol = False):
data = {}
DS = SCR
from vizgrimoire.GrimoireUtils import fill_and_order_items
type_analysis = None
if filter_ is not None:
type_analysis = [filter_.get_name(), filter_.get_item()]
from vizgrimoire.report import Report
automator = Report.get_config()
if evol:
metrics_on = DS.get_metrics_core_ts()
automator_metrics = DS.get_name()+"_metrics_ts"
else:
metrics_on = DS.get_metrics_core_agg()
automator_metrics = DS.get_name()+"_metrics_agg"
if automator_metrics in automator['r']:
metrics_on = automator['r'][automator_metrics].split(",")
logging.info(automator_metrics + " found ")
# print(metrics_on)
people_out = []
if "people_out" in Report.get_config()['r']:
people_out = Report.get_config()['r']["people_out"]
people_out = people_out.split(",")
metrics_reports = SCR.get_metrics_core_reports()
if filter_ is None:
from vizgrimoire.report import Report
reports_on = Report.get_config()['r']['reports'].split(",")
for r in metrics_reports:
if r in reports_on: metrics_on += [r]
if type_analysis and type_analysis[1] is None:
items = DS.get_filter_items(filter_, startdate, enddate, identities_db)
items = items.pop('name')
if DS.get_name()+"_start_date" in Report.get_config()['r']:
startdate = "'"+Report.get_config()['r'][DS.get_name()+"_start_date"]+"'"
if DS.get_name()+"_end_date" in Report.get_config()['r']:
enddate = "'"+Report.get_config()['r'][DS.get_name()+"_end_date"]+"'"
mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None)
all_metrics = SCR.get_metrics_set(SCR)
# SCR specific: remove some metrics from filters
if filter_ is not None:
metrics_not_filters = SCR.get_metrics_not_filters()
metrics_on_filters = list(set(metrics_on) - set(metrics_not_filters))
if filter_.get_name() == "repository":
if 'review_time' in metrics_on: metrics_on_filters+= ['review_time']
if 'submitted' in metrics_on: metrics_on_filters+= ['submitted']
metrics_on = metrics_on_filters
# END SCR specific
for item in all_metrics:
if item.id not in metrics_on: continue
# logging.info(item.id)
mfilter_orig = item.filters
mfilter.global_filter = mfilter_orig.global_filter
item.filters = mfilter
if not evol: mvalue = item.get_agg()
else: mvalue = item.get_ts()
if type_analysis and type_analysis[1] is None:
logging.info(item.id)
id_field = SCRQuery.get_group_field_alias(type_analysis[0])
mvalue = check_array_values(mvalue)
mvalue = fill_and_order_items(items, mvalue, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
# SCR SPECIFIC #
if evol:
if type_analysis and type_analysis[1] is None: pass
else:
metrics_on_changes = ['merged','abandoned','new']
for item in all_metrics:
if item.id in metrics_on_changes and filter_ is None:
mfilter_orig = item.filters
item.filters = mfilter
mvalue = item.get_ts_changes()
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
# END SCR SPECIFIC #
if not evol:
init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis)
end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis)
if type_analysis and type_analysis[1] is None:
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
init_date = fill_and_order_items(items, init_date, id_field,
evol, period, startdate, enddate)
end_date = fill_and_order_items(items, end_date, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + init_date.items() + end_date.items())
# Tendencies
metrics_trends = SCR.get_metrics_core_trends()
automator_metrics = DS.get_name()+"_metrics_trends"
if automator_metrics in automator['r']:
metrics_trends = automator['r'][automator_metrics].split(",")
for i in [7,30,365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
mfilter_orig = item.filters
item.filters = mfilter
period_data = item.get_trends(enddate, i)
item.filters = mfilter_orig
data = dict(data.items() + period_data.items())
if type_analysis and type_analysis[1] is None:
id_field = SCRQuery.get_group_field_alias(type_analysis[0])
period_data = fill_and_order_items(items, period_data, id_field)
data = dict(data.items() + period_data.items())
if filter_ is not None: studies_data = {}
else:
studies_data = DataSource.get_studies_data(SCR, period, startdate, enddate, evol)
data = dict(data.items() + studies_data.items())
return data
# env vars for R
os.environ["LANG"] = ""
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
logging.info("Starting Report analysis")
opts = read_options()
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " +
opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
def get_top_data(cls, startdate, enddate, identities_db, filter_, npeople):
bots = cls.get_bots()
closed_condition = cls._get_closed_condition()
# TODO: It should be configurable from Automator
top_issues_on = False
top = None
mopeners = DataSource.get_metrics("openers", cls)
mclosers = DataSource.get_metrics("closers", cls)
# We should check this metric is ON
stories_openers = DataSource.get_metrics("stories_openers", cls)
if mopeners is None or mclosers is None: return None
period = None
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
mfilter = MetricFilters(period, startdate, enddate, type_analysis,
npeople)
if mclosers.filters.closed_condition is not None:
mfilter.closed_condition = mclosers.filters.closed_condition
if filter_ is None:
top_closers_data = {}
top_closers_data['closers.'] = mclosers.get_list(mfilter, 0)
top_closers_data['closers.last month'] = mclosers.get_list(
mfilter, 31)
top_closers_data['closers.last year'] = mclosers.get_list(
mfilter, 365)
top_openers_data = {}
top_openers_data['openers.'] = mopeners.get_list(mfilter, 0)
top_openers_data['openers.last month'] = mopeners.get_list(
mfilter, 31)
top_openers_data['openers.last year'] = mopeners.get_list(
mfilter, 365)
top = dict(top_closers_data.items() + top_openers_data.items())
if top_issues_on:
from vizgrimoire.analysis.top_issues import TopIssues
from vizgrimoire.report import Report
db_identities = Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
dbname = Report.get_config()['generic'][cls.get_db_name()]
dbcon = ITSQuery(dbuser, dbpass, dbname, db_identities)
metric_filters = MetricFilters(None, startdate, enddate, [])
top_issues_data = TopIssues(dbcon, metric_filters).result(cls)
top = dict(top.items() + top_issues_data.items())
if False and stories_openers is not None:
top_sopeners_data = {}
top_sopeners_data[
'stories_openers.'] = stories_openers.get_list(mfilter, 0)
top_sopeners_data[
'stories_openers.last month'] = stories_openers.get_list(
mfilter, 31)
top_sopeners_data[
'stories_openers.last year'] = stories_openers.get_list(
mfilter, 365)
top = dict(top.items() + top_sopeners_data.items())
else:
filter_name = filter_.get_name()
if filter_name in ["company", "domain", "repository"]:
if filter_name in ["company", "domain", "repository"]:
top = {}
top['closers.'] = mclosers.get_list(mfilter, 0)
top['closers.last month'] = mclosers.get_list(mfilter, 31)
top['closers.last year'] = mclosers.get_list(mfilter, 365)
else:
# Remove filters above if there are performance issues
top = mclosers.get_list(mfilter)
else:
top = None
return top
def get_metrics_data(DS,
period,
startdate,
enddate,
identities_db,
filter_=None,
evol=False):
""" Get basic data from all core metrics """
from vizgrimoire.GrimoireUtils import fill_and_order_items
from vizgrimoire.ITS import ITS
from vizgrimoire.MLS import MLS
data = {}
dsquery = DSQuery
if DS == ITS: dsquery = ITSQuery
if DS == MLS: dsquery = MLSQuery
from vizgrimoire.report import Report
automator = Report.get_config()
if evol:
metrics_on = DS.get_metrics_core_ts()
automator_metrics = DS.get_name() + "_metrics_ts"
else:
metrics_on = DS.get_metrics_core_agg()
automator_metrics = DS.get_name() + "_metrics_agg"
if automator_metrics in automator['r']:
metrics_on = automator['r'][automator_metrics].split(",")
people_out = []
if "people_out" in Report.get_config()['r']:
people_out = Report.get_config()['r']["people_out"]
people_out = people_out.split(",")
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
if type_analysis and type_analysis[1] is None:
# We need the items for filling later values in group by queries
items = DS.get_filter_items(filter_, startdate, enddate,
identities_db)
if items is None: return data
items = items.pop('name')
if DS.get_name() + "_startdate" in Report.get_config()['r']:
startdate = Report.get_config()['r'][DS.get_name() + "_startdate"]
if DS.get_name() + "_enddate" in Report.get_config()['r']:
enddate = Report.get_config()['r'][DS.get_name() + "_enddate"]
# TODO: the hardcoded 10 should be removed, and use instead the npeople provided
# in the config file.
mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10,
people_out, None)
metrics_reports = DS.get_metrics_core_reports()
all_metrics = DS.get_metrics_set(DS)
# Reports = filters metrics not available inside filters
if type_analysis is None:
from vizgrimoire.report import Report
reports_on = Report.get_config()['r']['reports'].split(",")
for r in metrics_reports:
if r in reports_on: metrics_on += [r]
for item in all_metrics:
# print item
if item.id not in metrics_on: continue
mfilter_orig = item.filters
mfilter.global_filter = mfilter_orig.global_filter
mfilter.set_closed_condition(mfilter_orig.closed_condition)
item.filters = mfilter
if evol: mvalue = item.get_ts()
else: mvalue = item.get_agg()
if type_analysis and type_analysis[1] is None and mvalue:
logging.info(item.id)
id_field = None
# Support for combined filters
for idf in mvalue.keys():
if "CONCAT(" in idf:
id_field = idf
break
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
mvalue = fill_and_order_items(items, mvalue, id_field, evol,
period, startdate, enddate)
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
if not evol:
init_date = DS.get_date_init(startdate, enddate, identities_db,
type_analysis)
end_date = DS.get_date_end(startdate, enddate, identities_db,
type_analysis)
if type_analysis and type_analysis[1] is None:
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
init_date = fill_and_order_items(items, init_date, id_field,
evol, period, startdate,
enddate)
end_date = fill_and_order_items(items, end_date, id_field,
evol, period, startdate,
enddate)
if init_date is None: init_date = {}
if end_date is None: end_date = {}
data = dict(data.items() + init_date.items() + end_date.items())
# Tendencies
metrics_trends = DS.get_metrics_core_trends()
automator_metrics = DS.get_name() + "_metrics_trends"
if automator_metrics in automator['r']:
metrics_trends = automator['r'][automator_metrics].split(",")
for i in [7, 30, 365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
mfilter_orig = item.filters
item.filters = mfilter
period_data = item.get_trends(enddate, i)
item.filters = mfilter_orig
if type_analysis and type_analysis[1] is None:
group_field = dsquery.get_group_field_alias(
type_analysis[0])
period_data = fill_and_order_items(
items, period_data, group_field)
data = dict(data.items() + period_data.items())
return data
def init_env():
# env vars for R
os.environ["LANG"] = ""
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s')
logging.info("Starting Report analysis")
opts = read_options()
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " + opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
period = getPeriod(automator['r']['period'])
def get_metrics_data(DS, period, startdate, enddate, identities_db,
filter_ = None, evol = False):
""" Get basic data from all core metrics """
from vizgrimoire.GrimoireUtils import fill_and_order_items
from vizgrimoire.ITS import ITS
from vizgrimoire.MLS import MLS
data = {}
dsquery = DSQuery
if DS == ITS: dsquery = ITSQuery
if DS == MLS: dsquery = MLSQuery
from vizgrimoire.report import Report
automator = Report.get_config()
if evol:
metrics_on = DS.get_metrics_core_ts()
automator_metrics = DS.get_name()+"_metrics_ts"
else:
metrics_on = DS.get_metrics_core_agg()
automator_metrics = DS.get_name()+"_metrics_agg"
if automator_metrics in automator['r']:
metrics_on = automator['r'][automator_metrics].split(",")
people_out = []
if "people_out" in Report.get_config()['r']:
people_out = Report.get_config()['r']["people_out"]
people_out = people_out.split(",")
type_analysis = None
if filter_ is not None:
type_analysis = filter_.get_type_analysis()
if type_analysis and type_analysis[1] is None:
# We need the items for filling later values in group by queries
items = DS.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: return data
items = items.pop('name')
if DS.get_name()+"_startdate" in Report.get_config()['r']:
startdate = Report.get_config()['r'][DS.get_name()+"_startdate"]
if DS.get_name()+"_enddate" in Report.get_config()['r']:
enddate = Report.get_config()['r'][DS.get_name()+"_enddate"]
# TODO: the hardcoded 10 should be removed, and use instead the npeople provided
# in the config file.
mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None)
metrics_reports = DS.get_metrics_core_reports()
all_metrics = DS.get_metrics_set(DS)
# Reports = filters metrics not available inside filters
if type_analysis is None:
from vizgrimoire.report import Report
reports_on = Report.get_config()['r']['reports'].split(",")
for r in metrics_reports:
if r in reports_on: metrics_on += [r]
for item in all_metrics:
# print item
if item.id not in metrics_on: continue
mfilter_orig = item.filters
mfilter.global_filter = mfilter_orig.global_filter
mfilter.set_closed_condition(mfilter_orig.closed_condition)
item.filters = mfilter
if evol: mvalue = item.get_ts()
else: mvalue = item.get_agg()
if type_analysis and type_analysis[1] is None and mvalue:
logging.info(item.id)
id_field = None
# Support for combined filters
for idf in mvalue.keys():
if "CONCAT(" in idf:
id_field = idf
break
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
mvalue = fill_and_order_items(items, mvalue, id_field,
evol, period, startdate, enddate)
data = dict(data.items() + mvalue.items())
item.filters = mfilter_orig
if not evol:
init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis)
end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis)
if type_analysis and type_analysis[1] is None:
if id_field is None:
id_field = dsquery.get_group_field_alias(type_analysis[0])
init_date = fill_and_order_items(items, init_date, id_field,
evol, period, startdate, enddate)
end_date = fill_and_order_items(items, end_date, id_field,
evol, period, startdate, enddate)
if init_date is None: init_date = {}
if end_date is None: end_date = {}
data = dict(data.items() + init_date.items() + end_date.items())
# Tendencies
metrics_trends = DS.get_metrics_core_trends()
automator_metrics = DS.get_name()+"_metrics_trends"
if automator_metrics in automator['r']:
metrics_trends = automator['r'][automator_metrics].split(",")
for i in [7,30,365]:
for item in all_metrics:
if item.id not in metrics_trends: continue
mfilter_orig = item.filters
item.filters = mfilter
period_data = item.get_trends(enddate, i)
item.filters = mfilter_orig
if type_analysis and type_analysis[1] is None:
group_field = dsquery.get_group_field_alias(type_analysis[0])
period_data = fill_and_order_items(items, period_data, group_field)
data = dict(data.items() + period_data.items())
return data