def test_create_people_report(self):
#period, startdate, enddate, identities_db):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
if ds.get_name() == "downloads": continue
fpeople = ds.get_top_people_file(ds.get_name())
people = ds.get_top_people(startdate, enddate, identities_db, opts.npeople)
test_json = os.path.join("json",fpeople)
self.assertTrue(self._compare_data(people, test_json))
for upeople_id in people :
evol_data = ds.get_person_evol(upeople_id, period, startdate, enddate,
identities_db, type_analysis = None)
fperson = ds().get_person_evol_file(upeople_id)
test_json = os.path.join("json",fperson)
self.assertTrue(self._compare_data(evol_data, test_json))
agg = ds.get_person_agg(upeople_id, startdate, enddate,
identities_db, type_analysis = None)
fperson = ds().get_person_agg_file(upeople_id)
test_json = os.path.join("json",fperson)
self.assertTrue(self._compare_data(agg, test_json))
def test_create_reports_studies(self):
opts = read_options()
period = getPeriod(opts.granularity)
destdir = os.path.join("data","json")
from vizgrimoire.metrics.metrics_filter import MetricFilters
db_identities= Report.get_config()['generic']['db_identities']
dbuser = Report.get_config()['generic']['db_user']
dbpass = Report.get_config()['generic']['db_password']
studies = Report.get_studies()
metric_filters = MetricFilters(period, startdate, enddate, [])
for ds in Report.get_data_sources():
ds_dbname = ds.get_db_name()
dbname = Report.get_config()['generic'][ds_dbname]
dsquery = ds.get_query_builder()
dbcon = dsquery(dbuser, dbpass, dbname, db_identities)
for study in studies:
# logging.info("Creating report for " + study.id + " for " + ds.get_name())
try:
obj = study(dbcon, metric_filters)
obj.create_report(ds, destdir)
files = obj.get_report_files(ds)
if len(files) > 0:
for file in files:
f_test_json = os.path.join("json", file)
f_report_json = os.path.join(destdir, file)
if obj.get_definition()['id'] == "contributors_new_gone":
# authors is a dict with revtime which changes every execution
pass
else:
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
data = obj.result(ds)
if data is None: continue
test_json = os.path.join("json",ds.get_name()+"_"+obj.get_definition()['id']+".json")
if obj.get_definition()['id'] == "top_issues":
# Include time field which changes every execution
continue
else: self.assertTrue(self._compare_data(data, test_json))
except TypeError:
traceback.print_exc(file=sys.stdout)
logging.info(study.id + " does no support complete standard API. Not used when no available.")
continue
def off_test_get_filter_summary (self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
if ds.get_name() not in ['scm','its','mls']:
continue
Report.connect_ds(ds)
for filter_ in Report.get_filters():
if (filter_.get_name() == "company"):
summary = ds.get_filter_summary(filter_, period, startdate,
enddate, identities_db, 10)
test_json = os.path.join("json",filter_.get_summary_filename(ds))
self.assertTrue(self._compare_data(summary, test_json))
def test_get_agg_data(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test without filters
for ds in Report.get_data_sources():
logging.info(ds.get_name() + " test_get_agg_data")
# Create the evolutionary data from dbs and check with test JSON
Report.connect_ds(ds)
ds_data = ds.get_agg_data (period, startdate,
enddate, identities_db)
test_json = os.path.join("json",ds().get_agg_filename())
self.assertTrue(self._compare_data(ds_data, test_json))
def test_create_agg_evol_filters_all_data(self):
opts = read_options()
npeople = opts.npeople
destdir = opts.destdir
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
supported_all = {
"scm":["people2","company","country","repository","domain","projects","company+country","company+project"],
"its":["people2","company","country","repository","domain","projects","company+country","company+project"],
"its_1":["people2"],
"mls":["people2","company","country","repository","domain","projects"],
"scr":["people2","company","country","repository","domain","projects"],
"mediawiki":["people2"],
"irc":["people2"],
"downloads":["people2"],
"qaforums":["people2"],
"releases":["people2"],
"pullpo":["people2"],
"eventizer":[]
}
for ds in Report.get_data_sources():
for filter_ in Report.get_filters():
if filter_.get_name() in supported_all[ds.get_name()]:
logging.info("---> Using new filter API")
ds.create_filter_report_all(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
# Static
ds_json = filter_.get_static_filename_all(ds())
if ds_json in ["scr-cou-all-evolutionary.json","scr-cou-all-static.json",
"scr-dom-all-evolutionary.json","scr-dom-all-static.json",
"its-dom-all-static.json"]:
# The order of elements change in each execution
continue
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
# Evol
ds_json = filter_.get_evolutionary_filename_all(ds())
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
def test_get_agg_evol_filters_data(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test for all filters
for ds in Report.get_data_sources():
Report.connect_ds(ds)
for filter_ in Report.get_filters():
filter_name = filter_.get_name()
if filter_name in ["company+country","company+project"]:
# GROUP BY only filters
continue
filter_name_short = filter_.get_name_short()
bots = ds.get_filter_bots(filter_)
items = ds.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
for item in items:
filter_item = Filter(filter_.get_name(), item)
item_file = item
if ds.get_name() in ["its","scr","pullpo"] :
item_file = item.replace("/","_")
elif ds.get_name() == "mls":
item_file = item.replace("/","_").replace("<","__").replace(">","___")
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "agg")
agg = ds.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-static.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(agg, test_json))
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "evol")
evol = ds.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-evolutionary.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(evol, test_json))
def test_create_evolutionary_report(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test without filters
for ds in Report.get_data_sources():
# Create the evolutionary data from dbs and check with test JSON
logging.info(ds.get_name() + " create_evolutionary_report")
Report.connect_ds(ds)
ds.create_evolutionary_report (period, startdate,
enddate, opts.destdir, identities_db)
ds_json = ds().get_evolutionary_filename()
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
opts = read_options()
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " + opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
period = getPeriod(automator['r']['period'])
logging.info("Period: " + period)
if 'people_number' in automator['generic']:
npeople = automator['generic']['people_number']
logging.info("Number of people: " + npeople)
opts.npeople = npeople
# TODO: hack because VizR library needs. Fix in lib in future
startdate = "'"+start_date+"'"
enddate = "'"+end_date+"'"
identities_db = automator['generic']['db_identities']
import logging
import sys
import vizgrimoire.GrimoireUtils, vizgrimoire.GrimoireSQL
from vizgrimoire.GrimoireUtils import createJSON, completePeriodIds
from vizgrimoire.GrimoireUtils import getPeriod
from vizgrimoire.SCM import GetPeopleListSCM
import vizgrimoire.People
from utils import read_options
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
logging.info("Starting People data source analysis")
opts = read_options()
period = getPeriod(opts.granularity)
reports = opts.reports.split(",")
# TODO: hack because VizR library needs. Fix in lib in future
startdate = "'" + opts.startdate + "'"
enddate = "'" + opts.enddate + "'"
# Working at the same time with VizR and VizPy yet
# vizr.SetDBChannel (database=opts.dbname, user=opts.dbuser, password=opts.dbpassword)
GrimoireSQL.SetDBChannel(database=opts.dbname,
user=opts.dbuser,
password=opts.dbpassword)
people_data = {}
people = GetPeopleListSCM(startdate, enddate)
people = people['pid']
limit = 100
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " +
opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
period = getPeriod(automator['r']['period'])
logging.info("Period: " + period)
if 'people_number' in automator['generic']:
npeople = automator['generic']['people_number']
logging.info("Number of people: " + npeople)
opts.npeople = npeople
# TODO: hack because VizR library needs. Fix in lib in future
startdate = "'" + start_date + "'"
enddate = "'" + end_date + "'"
identities_db = automator['generic']['db_identities']
import sys
import vizgrimoire.GrimoireUtils, vizgrimoire.GrimoireSQL
from vizgrimoire.GrimoireUtils import createJSON, completePeriodIds
from vizgrimoire.GrimoireUtils import getPeriod
from vizgrimoire.SCM import GetPeopleListSCM
import vizgrimoire.People
from utils import read_options
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s')
logging.info("Starting People data source analysis")
opts = read_options()
period = getPeriod(opts.granularity)
reports = opts.reports.split(",")
# TODO: hack because VizR library needs. Fix in lib in future
startdate = "'"+opts.startdate+"'"
enddate = "'"+opts.enddate+"'"
# Working at the same time with VizR and VizPy yet
# vizr.SetDBChannel (database=opts.dbname, user=opts.dbuser, password=opts.dbpassword)
GrimoireSQL.SetDBChannel (database=opts.dbname, user=opts.dbuser, password=opts.dbpassword)
people_data = {}
people = GetPeopleListSCM(startdate, enddate)
people = people['pid']
limit = 100
if (len(people)<limit): limit = len(people);
people = people[0:limit]