def test_create_people_report(self):
#period, startdate, enddate, identities_db):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
if ds.get_name() == "downloads": continue
fpeople = ds.get_top_people_file(ds.get_name())
people = ds.get_top_people(startdate, enddate, identities_db, opts.npeople)
test_json = os.path.join("json",fpeople)
self.assertTrue(self._compare_data(people, test_json))
for upeople_id in people :
evol_data = ds.get_person_evol(upeople_id, period, startdate, enddate,
identities_db, type_analysis = None)
fperson = ds().get_person_evol_file(upeople_id)
test_json = os.path.join("json",fperson)
self.assertTrue(self._compare_data(evol_data, test_json))
agg = ds.get_person_agg(upeople_id, startdate, enddate,
identities_db, type_analysis = None)
fperson = ds().get_person_agg_file(upeople_id)
test_json = os.path.join("json",fperson)
self.assertTrue(self._compare_data(agg, test_json))
def test_get_filter_items(self):
opts = read_options()
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
for filter_ in Report.get_filters():
items = ds.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
if ds.get_name() in ["scr","pullpo"] :
items = [item.replace("/","_") for item in items]
elif ds.get_name() == "mls":
items = [item.replace("/","_").replace("<","__").replace(">","___")
for item in items]
fn = ds.get_name()+"-"+filter_.get_name_plural()+".json"
createJSON(items, opts.destdir+"/"+ fn)
test_json = os.path.join("json",fn)
new_json = opts.destdir+"/"+ fn
if ds.get_name() not in ["scr","pullpo"] :
# scr, pullpo repos format is more complex and
# is checked already in test_get_agg_evol_filters_data
self.assertTrue(self.compareJSON(test_json, new_json))
def test_get_filter_item_top (self):
opts = read_options()
npeople = opts.npeople
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
if ds.get_name() not in ['scm','its','mls']:
continue
Report.connect_ds(ds)
bots = ds.get_bots()
for filter_ in Report.get_filters():
items = ds.get_filter_items(filter_, startdate, enddate,
identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
for item in items:
filter_item = Filter(filter_.get_name(), item)
top = ds.get_top_data(startdate, enddate, identities_db,
filter_item, npeople)
if top is None or top == {}: continue
test_json = os.path.join("json",filter_item.get_top_filename(ds()))
self.assertTrue(self._compare_data(top, test_json))
def init(automator_file, metrics_path=None):
Report._automator_file = automator_file
Report._automator = read_main_conf(automator_file)
Report._init_filters()
Report._init_data_sources()
if metrics_path is not None:
Report._init_metrics(metrics_path)
studies_path = metrics_path.replace("metrics", "analysis")
Report._init_studies(studies_path)
def test_get_top_data (self):
opts = read_options()
npeople = opts.npeople
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
top = ds.get_top_data(startdate, enddate, identities_db, None, npeople)
test_json = os.path.join("json",ds.get_name()+"-top.json")
self.assertTrue(self._compare_data(top, test_json))
def test_create_top_report (self):
opts = read_options()
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
Report.connect_ds(ds)
ds.create_top_report(startdate, enddate, opts.destdir, opts.npeople, identities_db)
fn = ds.get_name()+"-top.json"
test_json = os.path.join("json",fn)
top_json = os.path.join(opts.destdir,fn)
self.assertTrue(self.compareJSON(test_json, top_json))
def off_test_get_filter_summary (self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
for ds in Report.get_data_sources():
if ds.get_name() not in ['scm','its','mls']:
continue
Report.connect_ds(ds)
for filter_ in Report.get_filters():
if (filter_.get_name() == "company"):
summary = ds.get_filter_summary(filter_, period, startdate,
enddate, identities_db, 10)
test_json = os.path.join("json",filter_.get_summary_filename(ds))
self.assertTrue(self._compare_data(summary, test_json))
def test_get_agg_data(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test without filters
for ds in Report.get_data_sources():
logging.info(ds.get_name() + " test_get_agg_data")
# Create the evolutionary data from dbs and check with test JSON
Report.connect_ds(ds)
ds_data = ds.get_agg_data (period, startdate,
enddate, identities_db)
test_json = os.path.join("json",ds().get_agg_filename())
self.assertTrue(self._compare_data(ds_data, test_json))
def test_create_agg_evol_filters_all_data(self):
opts = read_options()
npeople = opts.npeople
destdir = opts.destdir
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
supported_all = {
"scm":["people2","company","country","repository","domain","projects","company+country","company+project"],
"its":["people2","company","country","repository","domain","projects","company+country","company+project"],
"its_1":["people2"],
"mls":["people2","company","country","repository","domain","projects"],
"scr":["people2","company","country","repository","domain","projects"],
"mediawiki":["people2"],
"irc":["people2"],
"downloads":["people2"],
"qaforums":["people2"],
"releases":["people2"],
"pullpo":["people2"],
"eventizer":[]
}
for ds in Report.get_data_sources():
for filter_ in Report.get_filters():
if filter_.get_name() in supported_all[ds.get_name()]:
logging.info("---> Using new filter API")
ds.create_filter_report_all(filter_, period, startdate, enddate,
destdir, npeople, identities_db)
# Static
ds_json = filter_.get_static_filename_all(ds())
if ds_json in ["scr-cou-all-evolutionary.json","scr-cou-all-static.json",
"scr-dom-all-evolutionary.json","scr-dom-all-static.json",
"its-dom-all-static.json"]:
# The order of elements change in each execution
continue
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
# Evol
ds_json = filter_.get_evolutionary_filename_all(ds())
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
def test_get_agg_evol_filters_data(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test for all filters
for ds in Report.get_data_sources():
Report.connect_ds(ds)
for filter_ in Report.get_filters():
filter_name = filter_.get_name()
if filter_name in ["company+country","company+project"]:
# GROUP BY only filters
continue
filter_name_short = filter_.get_name_short()
bots = ds.get_filter_bots(filter_)
items = ds.get_filter_items(filter_, startdate, enddate, identities_db)
if items is None: continue
if (isinstance(items, dict)): items = items['name']
if not isinstance(items, (list)): items = [items]
for item in items:
filter_item = Filter(filter_.get_name(), item)
item_file = item
if ds.get_name() in ["its","scr","pullpo"] :
item_file = item.replace("/","_")
elif ds.get_name() == "mls":
item_file = item.replace("/","_").replace("<","__").replace(">","___")
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "agg")
agg = ds.get_agg_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-static.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(agg, test_json))
logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "evol")
evol = ds.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item)
fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-evolutionary.json"
test_json = os.path.join("json",fn)
self.assertTrue(self._compare_data(evol, test_json))
def test_create_evolutionary_report(self):
opts = read_options()
period = getPeriod(opts.granularity)
automator = read_main_conf(opts.config_file)
identities_db = automator['generic']['db_identities']
# Test without filters
for ds in Report.get_data_sources():
# Create the evolutionary data from dbs and check with test JSON
logging.info(ds.get_name() + " create_evolutionary_report")
Report.connect_ds(ds)
ds.create_evolutionary_report (period, startdate,
enddate, opts.destdir, identities_db)
ds_json = ds().get_evolutionary_filename()
f_test_json = os.path.join("json", ds_json)
f_report_json = os.path.join(opts.destdir, ds_json)
self.assertTrue(self.compareJSON(f_test_json, f_report_json))
os.environ["LANG"] = ""
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s')
logging.info("Starting Report analysis")
opts = read_options()
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " + opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
period = getPeriod(automator['r']['period'])
logging.info("Period: " + period)
os.environ["R_LIBS"] = "../../r-lib"
if __name__ == '__main__':
init_env()
from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON
from vizgrimoire.report import Report
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
logging.info("Starting Report analysis")
opts = read_options()
Report.init(opts.config_file, opts.metrics_path)
automator = read_main_conf(opts.config_file)
if 'start_date' not in automator['r']:
logging.error("start_date (yyyy-mm-dd) not found in " +
opts.config_file)
sys.exit()
start_date = automator['r']['start_date']
if 'end_date' not in automator['r']:
end_date = time.strftime('%Y-%m-%d')
else:
end_date = automator['r']['end_date']
if 'period' not in automator['r']:
period = getPeriod("months")
else:
period = getPeriod(automator['r']['period'])
logging.info("Period: " + period)
