def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = QAForums.get_filter_items(filter_, startdate, enddate, identities_db)
if items == None:
return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(QAForums()))
createJSON(file_items, fn)
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = QAForums.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(QAForums()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = QAForums.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(QAForums()))
createJSON(agg, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items:
item = item.replace("'", "\\'")
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db,
filter_item, npeople, False)
createJSON(top_senders,
destdir + "/" + filter_item.get_top_filename(MLS()))
def test_get_filter_item_top (self):
opts = read_options()
npeople = opts.npeople
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
if ds.get_name() not in ['scm','its','mls']:
continue
Report.connect_ds(ds)
bots = ds.get_bots()
for filter_ in Report.get_filters():
items = ds.get_filter_items(filter_, startdate, enddate,
identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
for item in items:
filter_item = Filter(filter_.get_name(), item)
top = ds.get_top_data(startdate, enddate, identities_db,
filter_item, npeople)
if top is None or top == {}: continue
test_json = os.path.join("json",filter_item.get_top_filename(ds()))
self.assertTrue(self._compare_data(top, test_json))
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir, filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("company","project","repository"):
top_mergers = SCR.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(SCR()))
createJSON(top_mergers, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items:
# item_name = "'"+ item+ "'"
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(IRC()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
items_files = [item.replace('/', '_').replace("<","__").replace(">","___")
for item in items]
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_files, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'sent_365' : [], 'senders_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_.get_name(), item)
evol_data = MLS.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(MLS()))
createJSON(evol_data, fn)
agg = MLS.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(MLS()))
createJSON(agg, fn)
if filter_name in ("domain", "company", "repository"):
items_list['name'].append(item.replace('/', '_').replace("<","__").replace(">","___"))
items_list['sent_365'].append(agg['sent_365'])
items_list['senders_365'].append(agg['senders_365'])
top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False)
createJSON(top_senders, destdir+"/"+filter_item.get_top_filename(MLS()))
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = MLS
if False:
summary = MLS.get_filter_summary(
filter_, period, startdate, enddate,
identities_db, 10, self.db.projects_db
)
createJSON (summary,
destdir + "/" + filter_.get_summary_filename(MLS))
# Perform ages study, if it is specified in Report
MLS.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = IRC.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(IRC()))
createJSON(items, fn)
for item in items :
# item_name = "'"+ item+ "'"
logging.info (item)
filter_item = Filter(filter_.get_name(), item)
evol_data = IRC.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(IRC()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = IRC.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(IRC()))
createJSON(agg, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items, fn)
for item in items:
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_name, item)
if filter_name in ("repository"):
logging.warn("Filter 'repository' detected for top info")
top_authors = EventsDS.get_top_data(startdate, enddate,
identities_db, filter_item,
npeople)
logging.warn(filter_item.get_top_filename(EventsDS()))
fn = os.path.join(destdir,
filter_item.get_top_filename(EventsDS()))
createJSON(top_authors, fn)
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = cls.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
createJSON(evol_data, fn)
agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(agg, fn)
if filter_name in ["domain", "company", "repository"]:
items_list['name'].append(item.replace('/', '_'))
items_list['closed_365'].append(agg['closed_365'])
items_list['closers_365'].append(agg['closers_365'])
if filter_name in ["company","domain","repository"]:
top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(cls()))
createJSON(top, fn)
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = ITS
summary = cls.get_filter_summary(
filter_, period, startdate, enddate,
identities_db, 10
)
createJSON (summary,
destdir + "/" + filter_.get_summary_filename(cls))
# Perform ages study, if it is specified in Report
cls.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = cls.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []}
else:
items_list = items
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
createJSON(evol_data, fn)
agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(agg, fn)
if filter_name in ["domain", "company", "repository"]:
items_list['name'].append(item.replace('/', '_'))
items_list['closed_365'].append(agg['closed_365'])
items_list['closers_365'].append(agg['closers_365'])
if filter_name in ["company","domain","repository"]:
top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople)
fn = os.path.join(destdir, filter_item.get_top_filename(cls()))
createJSON(top, fn)
fn = os.path.join(destdir, filter_.get_filename(cls()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = ITS
#summary = cls.get_filter_summary(
# filter_, period, startdate, enddate,
# identities_db, 10
# )
#createJSON (summary,
# destdir + "/" + filter_.get_summary_filename(cls))
# Perform ages study, if it is specified in Report
cls.ages_study_com (items, period, startdate, enddate, destdir)
def ages_study_com (ds, items, period,
startdate, enddate, destdir):
"""Perform ages study for companies, if it is specified in Report.
Produces JSON files for those studies.
Parameters
----------
ds: { SCM | ITS | MLS }
Data source
items: ??
Items
period: ??
Period
startdate: ??
Start date
enddate: ??
End date
destdir: string
Directory for writing the JSON files
"""
from vizgrimoire.report import Report
filter_name = "company"
studies = Report.get_studies()
ages = None
for study in studies:
if study.id == "ages":
ages = study
if ages is not None:
# Get config parameters for producing a connection
# to the database
config = Report.get_config()
db_identities = config['generic']['db_identities']
dbuser = config['generic']['db_user']
dbpass = config['generic']['db_password']
start_string = ds.get_name() + "_start_date"
end_string = ds.get_name() + "_end_date"
if start_string in config['r']:
startdate = "'" + config['r'][start_string] + "'"
if end_string in config['r']:
enddate = "'" + config['r'][end_string] + "'"
ds_dbname = ds.get_db_name()
dbname = config['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for item in items :
filter_item = Filter(filter_name, item)
metric_filters = MetricFilters(
period, startdate, enddate,
filter_item.get_type_analysis()
)
obj = ages(dbcon, metric_filters)
res = obj.create_report(ds, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
# Include metrics to sort in javascript.
items_list = {
"name": [],
"review_time_days_median": [],
"submitted": []
}
for item in items:
item_file = item.replace("/", "_")
items_list["name"].append(item_file)
logging.info(item)
filter_item = Filter(filter_name, item)
evol = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(SCR()))
createJSON(evol, fn)
# Static
agg = SCR.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCR()))
createJSON(agg, fn)
if 'submitted' in agg:
items_list["submitted"].append(agg["submitted"])
else:
items_list["submitted"].append("NA")
if 'review_time_days_median' in agg:
items_list["review_time_days_median"].append(
agg['review_time_days_median'])
else:
items_list["review_time_days_median"].append("NA")
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items_list, fn)
SCR.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_" + item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name': [], 'events_365': [], 'rsvps_365': []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate,
enddate, identities_db,
filter_item)
fn = os.path.join(
destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def ages_study_com(ds, items, period, startdate, enddate, destdir):
"""Perform ages study for companies, if it is specified in Report.
Produces JSON files for those studies.
Parameters
----------
ds: { SCM | ITS | MLS }
Data source
items: ??
Items
period: ??
Period
startdate: ??
Start date
enddate: ??
End date
destdir: string
Directory for writing the JSON files
"""
from vizgrimoire.report import Report
filter_name = "company"
studies = Report.get_studies()
ages = None
for study in studies:
if study.id == "ages":
ages = study
if ages is not None:
# Get config parameters for producing a connection
# to the database
config = Report.get_config()
db_identities = config['generic']['db_identities']
dbuser = config['generic']['db_user']
dbpass = config['generic']['db_password']
start_string = ds.get_name() + "_start_date"
end_string = ds.get_name() + "_end_date"
if start_string in config['r']:
startdate = "'" + config['r'][start_string] + "'"
if end_string in config['r']:
enddate = "'" + config['r'][end_string] + "'"
ds_dbname = ds.get_db_name()
dbname = config['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for item in items:
filter_item = Filter(filter_name, item)
metric_filters = MetricFilters(period, startdate, enddate,
filter_item.get_type_analysis())
obj = ages(dbcon, metric_filters)
res = obj.create_report(ds, destdir)
def create_filter_report_all(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
check = False # activate to debug pull_requests
filter_name = filter_.get_name()
if filter_name == "people2" or filter_name == "company+country":
filter_all = Filter(filter_name, None)
agg_all = Pullpo.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_static_filename_all(Pullpo()))
createJSON(agg_all, fn)
evol_all = Pullpo.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_evolutionary_filename_all(Pullpo()))
createJSON(evol_all, fn)
if check:
Pullpo._check_report_all_data(evol_all, filter_, startdate,
enddate, identities_db, True,
period)
Pullpo._check_report_all_data(agg_all, filter_, startdate,
enddate, identities_db, False,
period)
else:
logging.error(filter_name +
" does not support yet group by items sql queries")
def get_list(self):
from vizgrimoire.data_source import DataSource
from vizgrimoire.filter import Filter
startdate = self.filters.startdate
enddate = self.filters.enddate
closed_condition = ITS._get_closed_condition()
if self.filters.closed_condition is not None:
closed_condition = self.filters.closed_condition
bots = DataSource.get_filter_bots(Filter("domain"))
fbots = ''
for bot in bots:
fbots += " dom.name<>'" + bot + "' and "
tables = Set([])
filters = Set([])
tables.union_update(self.db.GetTablesDomains(self.db.identities_db))
tables_str = self.db._get_tables_query(tables)
filters.union_update(self.db.GetFiltersDomains())
filters_str = self.db._get_filters_query(filters)
q = "SELECT DISTINCT(SUBSTR(email,LOCATE('@',email)+1)) as domain "+\
"FROM "+ tables_str + " "+\
"WHERE " + filters_str +" AND "+\
" "+ fbots +" "+\
" c.changed_on >= "+ startdate+ " AND "+\
" c.changed_on < "+ enddate+ " AND "+\
" "+ closed_condition+" "+\
"GROUP BY domain "+\
"ORDER BY COUNT(DISTINCT(c.issue_id)) DESC, domain LIMIT " + str(Metrics.domains_limit)
data = self.db.ExecuteQuery(q)
data['name'] = data.pop('domain')
return (data)
def create_filter_report_all(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
if filter_name in ["people2","company", "country","repository","domain","project",
"company"+MetricFilters.DELIMITER+"country",
"company"+MetricFilters.DELIMITER+"project"] :
filter_all = Filter(filter_name, None)
agg_all = cls.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(cls()))
createJSON(agg_all, fn)
ITS.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = cls.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(cls()))
createJSON(evol_all, fn)
ITS.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
cls._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
cls._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db):
check = False # activate to debug issues
filter_name = filter_.get_name()
if filter_name in ["people2","company","repository","country","domain","project"] :
filter_all = Filter(filter_name, None)
agg_all = SCR.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(SCR()))
createJSON(agg_all, fn)
SCR.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(SCR()))
createJSON(evol_all, fn)
SCR.convert_all_to_single(evol_all, filter_, destdir, True, period)
if check:
SCR._check_report_all_data(evol_all, filter_, startdate, enddate,
identities_db, True, period)
SCR._check_report_all_data(agg_all, filter_, startdate, enddate,
identities_db, False, period)
else:
logging.error(filter_name +" does not support yet group by items sql queries")
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
filter_name = filter_.get_name()
items = items['name']
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_"+item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(file_items, fn)
if filter_name in ("repository"):
items_list = {'name' : [], 'events_365' : [], 'rsvps_365' : []}
else:
items_list = items
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = EventsDS.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(EventsDS()))
createJSON(completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = EventsDS.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(EventsDS()))
createJSON(agg, fn)
if filter_name in ("repository"):
items_list['name'].append(item.replace('/', '_'))
items_list['events_365'].append(agg['events_365'])
items_list['rsvps_365'].append(agg['rsvps_365'])
EventsDS.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
fn = os.path.join(destdir, filter_.get_filename(EventsDS()))
createJSON(items_list, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCM.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCM()))
escaped_items = [i.replace('/', '_') for i in items]
logging.info("D1")
createJSON(escaped_items, fn)
for item in items:
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_name, item)
evol_data = SCM.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(SCM()))
createJSON(evol_data, fn)
agg = SCM.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCM()))
createJSON(agg, fn)
SCM.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com(items, period, startdate, enddate, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCR.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
# Include metrics to sort in javascript.
items_list = {"name":[],"review_time_days_median":[],"submitted":[]}
for item in items :
item_file = item.replace("/","_")
items_list["name"].append(item_file)
logging.info (item)
filter_item = Filter(filter_name, item)
evol = SCR.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCR()))
createJSON(evol, fn)
# Static
agg = SCR.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCR()))
createJSON(agg, fn)
if 'submitted' in agg:
items_list["submitted"].append(agg["submitted"])
else: items_list["submitted"].append("NA")
if 'review_time_days_median' in agg:
items_list["review_time_days_median"].append(agg['review_time_days_median'])
else: items_list["review_time_days_median"].append("NA")
fn = os.path.join(destdir, filter_.get_filename(SCR()))
createJSON(items_list, fn)
SCR.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
def create_filter_report_all(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
# New API for getting all metrics with one query
check = False # activate to debug issues
filter_name = filter_.get_name()
# Change filter to GrimoireLib notation
filter_name = filter_name.replace("+", MetricFilters.DELIMITER)
# This report is created per item, not using GROUP BY yet
SCM.create_filter_report_top(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
# Filters metrics computed using GROUP BY queries
if filter_name in [
"people2", "company", "country", "repository", "domain",
"project", "company" + MetricFilters.DELIMITER + "country",
"company" + MetricFilters.DELIMITER + "project"
]:
filter_all = Filter(filter_name, None)
agg_all = SCM.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(SCM()))
createJSON(agg_all, fn)
SCM.convert_all_to_single(agg_all, filter_, destdir, False, period)
evol_all = SCM.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_evolutionary_filename_all(SCM()))
createJSON(evol_all, fn)
SCM.convert_all_to_single(evol_all, filter_, destdir, True, period)
# Studies report for filters
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com(agg_all['name'], period, startdate, enddate,
destdir)
# check is only done fpr basic filters. Composed should work if basic does.
if check and not "," in filter_name:
SCM._check_report_all_data(evol_all, filter_, startdate,
enddate, identities_db, True,
period)
SCM._check_report_all_data(agg_all, filter_, startdate,
enddate, identities_db, False,
period)
else:
logging.error(filter_name +
" does not support yet group by items sql queries")
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
for item in items :
item = item.replace("'", "\\'")
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_.get_name(), item)
top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False)
createJSON(top_senders, destdir+"/"+filter_item.get_top_filename(MLS()))
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = QAForums.get_filter_items(filter_, startdate, enddate,
identities_db)
if items == None:
return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, list):
items = [items]
file_items = []
for item in items:
if re.compile("^\..*").match(item) is not None: item = "_" + item
file_items.append(item)
fn = os.path.join(destdir, filter_.get_filename(QAForums()))
createJSON(file_items, fn)
for item in items:
logging.info(item)
filter_item = Filter(filter_.get_name(), item)
evol_data = QAForums.get_evolutionary_data(period, startdate,
enddate, identities_db,
filter_item)
fn = os.path.join(
destdir, filter_item.get_evolutionary_filename(QAForums()))
createJSON(
completePeriodIds(evol_data, period, startdate, enddate), fn)
agg = QAForums.get_agg_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_static_filename(QAForums()))
createJSON(agg, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = SCM.get_filter_items(filter_, startdate, enddate, identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
fn = os.path.join(destdir, filter_.get_filename(SCM()))
escaped_items = [i.replace('/','_') for i in items]
logging.info("D1")
createJSON(escaped_items, fn)
for item in items :
item_name = "'"+ item+ "'"
logging.info (item_name)
filter_item = Filter(filter_name, item)
evol_data = SCM.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCM()))
createJSON(evol_data, fn)
agg = SCM.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(SCM()))
createJSON(agg, fn)
SCM.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
if (filter_name == "company"):
ds = SCM
#summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10)
#createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM))
# Perform ages study, if it is specified in Report
SCM.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report_all(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
filter_name = filter_.get_name()
if filter_name == "people2" or filter_name == "company_off":
filter_all = Filter(filter_name, None)
agg_all = IRC.get_agg_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir, filter_.get_static_filename_all(IRC()))
createJSON(agg_all, fn)
evol_all = IRC.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_all)
fn = os.path.join(destdir,
filter_.get_evolutionary_filename_all(IRC()))
createJSON(evol_all, fn)
else:
logging.error(IRC.get_name() + " " + filter_name +
" does not support yet group by items sql queries")
def test_get_agg_evol_filters_data(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test for all filters
for ds in Report.get_data_sources():
Report.connect_ds(ds)
for filter_ in Report.get_filters():
filter_name = filter_.get_name()
if filter_name in ["company+country","company+project"]:
# GROUP BY only filters
continue
filter_name_short = filter_.get_name_short()
bots = ds.get_filter_bots(filter_)
items = ds.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
for item in items:
filter_item = Filter(filter_.get_name(), item)
item_file = item
if ds.get_name() in ["its","scr","pullpo"] :
item_file = item.replace("/","_")
elif ds.get_name() == "mls":
item_file = item.replace("/","_").replace("<","__").replace(">","___")
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "agg")
agg = ds.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-static.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(agg, test_json))
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "evol")
evol = ds.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-evolutionary.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(evol, test_json))
def get_list(self):
filter_ = DataSource.get_filter_bots(Filter("country"))
filter_countries = ''
for country in filter_:
filter_countries += " cou.name<>'" + country + "' AND "
countries_tables = self.db._get_tables_query(
self.db.GetTablesCountries())
countries_filters = self.db._get_filters_query(
self.db.GetFiltersCountries())
q = "SELECT cou.name as name, COUNT(m.message_ID) as sent "+\
"FROM "+ countries_tables + " "+\
"WHERE "+ countries_filters + " AND "+\
" "+ filter_countries+ " "+\
" m.first_date >= "+self.filters.startdate+" AND "+\
" m.first_date < "+self.filters.enddate+" "+\
"GROUP BY cou.name "+\
"ORDER BY COUNT((m.message_ID)) DESC, cou.name "
data = self.db.ExecuteQuery(q)
return (data['name'])
def _init_filters():
reports = Report._automator['r']['reports']
# Hack because we use repos in filters
reports = reports.replace("repositories", "repos")
filters = reports.split(",")
for name in filters:
filter_ = Filter.get_filter_from_plural(name)
if filter_ is not None:
if filter_.get_name() == "project":
if not 'db_projects' in Report._automator['generic']:
logging.error(
"project filter configured but no db_projects. Review "
+ Report._automator_file)
raise Exception("Wrong configuration")
Report._filters.append(filter_)
Report._filters_automator.append(filter_)
else:
logging.error("Wrong filter " + name + ". Review " +
Report._automator_file)
raise Exception('Wrong automator config file')
def _check_report_all_data(data,
filter_,
startdate,
enddate,
idb,
evol=False,
period=None):
# Check per item data with group by people data
items = SCM.get_filter_items(filter_, startdate, enddate, idb)
id_field = DSQuery.get_group_field(filter_.get_name())
id_field = id_field.split('.')[1] # remove table name
for i in range(0, len(items['name'])):
name = items['name'][i]
logging.info("Checking " + name + " " + str(i) + "/" +
str(len(items['name'])))
if filter_.get_name() == "people2":
uuid = items['id'][i]
item = uuid
else:
item = name
pos = data[id_field].index(name)
type_analysis = [filter_.get_name(), item]
filter_item = Filter(filter_.get_name(), item)
if not evol:
if filter_.get_name() == "people2":
agg = SCM.get_person_agg(uuid, startdate, enddate, idb,
type_analysis)
else:
agg = SCM.get_agg_data(period, startdate, enddate, idb,
filter_item)
assert agg['commits'] == data['commits'][pos]
else:
if filter_.get_name() == "people2":
ts = SCM.get_person_evol(uuid, period, startdate, enddate,
idb, type_analysis)
else:
ts = SCM.get_evolutionary_data(period, startdate, enddate,
idb, filter_item)
assert ts['commits'] == data['commits'][pos]
def get_list(self):
filter_ = DataSource.get_filter_bots(Filter("company"))
filter_organizations = ''
for company in filter_:
filter_organizations += " org.name<>'" + company + "' AND "
organizations_tables = self.db._get_tables_query(
self.db.GetTablesCompanies())
organizations_filters = self.db._get_filters_query(
self.db.GetFiltersCompanies())
q = "SELECT org.name as name, COUNT(DISTINCT(m.message_ID)) as sent "+\
" FROM "+ organizations_tables + " "+\
" WHERE "+ organizations_filters + " AND "+\
" "+ filter_organizations+ " "+\
" m.first_date >= "+self.filters.startdate+" AND "+\
" m.first_date < "+self.filters.enddate+" "+\
" GROUP BY org.name "+\
" ORDER BY COUNT(DISTINCT(m.message_ID)) DESC"
data = self.db.ExecuteQuery(q)
return (data['name'])
def get_list(self):
from vizgrimoire.data_source import DataSource
from vizgrimoire.filter import Filter
bots = DataSource.get_filter_bots(Filter("company"))
fbots = ''
for bot in bots:
fbots += " org.name<>'"+bot+"' and "
startdate = self.filters.startdate
enddate = self.filters.enddate
closed_condition = ITS._get_closed_condition()
if self.filters.closed_condition is not None:
closed_condition = self.filters.closed_condition
# list each of the organizations analyzed
# those are order by number of closed issues
q = "select org.name "+\
"from issues i, "+\
" changes ch, "+\
" people_uidentities pup, "+\
" "+ self.db.identities_db+ ".enrollments enr, "+\
" "+ self.db.identities_db+ ".organizations org "+\
"where i.id = ch.issue_id and "+\
" ch.changed_by = pup.people_id and "+\
" pup.uuid = enr.uuid and "+\
" enr.organization_id = org.id and "+\
" ch.changed_on >= "+ startdate+ " and "+\
" ch.changed_on < "+ enddate+" and "+\
" i.submitted_on >= enr.start and "+\
" i.submitted_on < enr.end and "+\
" "+ fbots +\
closed_condition +\
" group by org.name "+\
" order by count(distinct(i.id)) desc"
data = self.db.ExecuteQuery(q)
return (data)
def convert_all_to_single(cls, data, filter_, destdir, evolutionary, period='month'):
""" Convert a GROUP BY result to follow tradition individual JSON files """
from vizgrimoire.SCM import SCM
from vizgrimoire.ITS import ITS
from vizgrimoire.SCR import SCR
from vizgrimoire.MLS import MLS
from vizgrimoire.filter import Filter
if cls == ITS or cls == SCR:
if 'url' in data.keys():
data['name'] = data.pop('url')
data['name'] = [item.replace('/', '_') for item in data['name']]
elif cls == MLS:
if 'mailing_list_url' in data.keys():
data['name'] = data.pop('mailing_list_url')
data['name'] = [item.replace('/', '_') for item in data['name']]
if cls == ITS or cls == SCM:
if filter_.get_name() in ["company+country","company+project"]:
data['name'] = data.pop('filter')
if not evolutionary:
# First create the JSON with the list of items
item_list = {}
fn = os.path.join(destdir, filter_.get_filename(cls))
if cls == SCM:
fields = ["authors_365","name","commits_365"]
elif cls == ITS:
fields = ["closed_365","closers_365", "name"]
elif cls == MLS:
fields = ["sent_365","senders_365", "name"]
elif cls == SCR:
fields = ["submitted","review_time_days_median","review_time_pending_upload_ReviewsWaitingForReviewer_days_median", "name"]
else:
fields = ["name"]
for field in fields:
if field in data:
item_list[field] = data[field]
createJSON(item_list, fn)
# Items files
ts_fields = ['unixtime','id','date',period]
# Not metrics fields
no_metrics_fields = ['filter_type']
for i in range(0,len(data['name'])):
item_metrics = {}
item = data['name'][i]
for metric in data:
if metric == "name": continue
if metric in ts_fields+no_metrics_fields: continue
if len(data[metric])<len(data['name']):
logging.error(cls.get_name()+" "+metric + " not supported in GROUP BY. Not included")
continue
if evolutionary:
if not isinstance(data[metric][i], list):
logging.error(cls.get_name()+" "+metric + " evol not supported in GROUP BY. Not included")
continue
item_metrics[metric] = data[metric][i]
filter_item = Filter(filter_.get_name(), item)
if evolutionary:
for field in ts_fields:
# Shared time series fields
item_metrics[field] = data[field]
fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls()))
else:
fn = os.path.join(destdir, filter_item.get_static_filename(cls()))
createJSON(item_metrics, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir,
npeople, identities_db):
from vizgrimoire.report import Report
items = Report.get_items()
if items is None:
items = MLS.get_filter_items(filter_, startdate, enddate,
identities_db)
if (items == None): return
items = items['name']
filter_name = filter_.get_name()
if not isinstance(items, (list)):
items = [items]
items_files = [
item.replace('/', '_').replace("<", "__").replace(">", "___")
for item in items
]
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_files, fn)
if filter_name in ("domain", "company", "repository"):
items_list = {'name': [], 'sent_365': [], 'senders_365': []}
else:
items_list = items
for item in items:
item = item.replace("'", "\\'")
item_name = "'" + item + "'"
logging.info(item_name)
filter_item = Filter(filter_.get_name(), item)
evol_data = MLS.get_evolutionary_data(period, startdate, enddate,
identities_db, filter_item)
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(MLS()))
createJSON(evol_data, fn)
agg = MLS.get_agg_data(period, startdate, enddate, identities_db,
filter_item)
fn = os.path.join(destdir, filter_item.get_static_filename(MLS()))
createJSON(agg, fn)
if filter_name in ("domain", "company", "repository"):
items_list['name'].append(
item.replace('/', '_').replace("<",
"__").replace(">", "___"))
items_list['sent_365'].append(agg['sent_365'])
items_list['senders_365'].append(agg['senders_365'])
top_senders = MLS.get_top_data(startdate, enddate, identities_db,
filter_item, npeople, False)
createJSON(top_senders,
destdir + "/" + filter_item.get_top_filename(MLS()))
fn = os.path.join(destdir, filter_.get_filename(MLS()))
createJSON(items_list, fn)
if (filter_name == "company"):
ds = MLS
if False:
summary = MLS.get_filter_summary(filter_, period, startdate,
enddate, identities_db, 10,
self.db.projects_db)
createJSON(summary,
destdir + "/" + filter_.get_summary_filename(MLS))
# Perform ages study, if it is specified in Report
MLS.ages_study_com(items, period, startdate, enddate, destdir)
def convert_all_to_single(cls,
data,
filter_,
destdir,
evolutionary,
period='month'):
""" Convert a GROUP BY result to follow tradition individual JSON files """
from vizgrimoire.SCM import SCM
from vizgrimoire.ITS import ITS
from vizgrimoire.SCR import SCR
from vizgrimoire.MLS import MLS
from vizgrimoire.filter import Filter
if cls == ITS or cls == SCR:
if 'url' in data.keys():
data['name'] = data.pop('url')
data['name'] = [
item.replace('/', '_') for item in data['name']
]
elif cls == MLS:
if 'mailing_list_url' in data.keys():
data['name'] = data.pop('mailing_list_url')
data['name'] = [
item.replace('/', '_') for item in data['name']
]
if cls == ITS or cls == SCM:
if filter_.get_name() in ["company+country", "company+project"]:
data['name'] = data.pop('filter')
if not evolutionary:
# First create the JSON with the list of items
item_list = {}
fn = os.path.join(destdir, filter_.get_filename(cls))
if cls == SCM:
fields = ["authors_365", "name", "commits_365"]
elif cls == ITS:
fields = ["closed_365", "closers_365", "name"]
elif cls == MLS:
fields = ["sent_365", "senders_365", "name"]
elif cls == SCR:
fields = [
"submitted", "review_time_days_median",
"review_time_pending_upload_ReviewsWaitingForReviewer_days_median",
"name"
]
else:
fields = ["name"]
for field in fields:
if field in data:
item_list[field] = data[field]
createJSON(item_list, fn)
# Items files
ts_fields = ['unixtime', 'id', 'date', period]
# Not metrics fields
no_metrics_fields = ['filter_type']
for i in range(0, len(data['name'])):
item_metrics = {}
item = data['name'][i]
for metric in data:
if metric == "name": continue
if metric in ts_fields + no_metrics_fields: continue
if len(data[metric]) < len(data['name']):
logging.error(cls.get_name() + " " + metric +
" not supported in GROUP BY. Not included")
continue
if evolutionary:
if not isinstance(data[metric][i], list):
logging.error(
cls.get_name() + " " + metric +
" evol not supported in GROUP BY. Not included")
continue
item_metrics[metric] = data[metric][i]
filter_item = Filter(filter_.get_name(), item)
if evolutionary:
for field in ts_fields:
# Shared time series fields
item_metrics[field] = data[field]
fn = os.path.join(destdir,
filter_item.get_evolutionary_filename(cls()))
else:
fn = os.path.join(destdir,
filter_item.get_static_filename(cls()))
createJSON(item_metrics, fn)
