def test_create_people_report(self): #period, startdate, enddate, identities_db): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): Report.connect_ds(ds) if ds.get_name() == "downloads": continue fpeople = ds.get_top_people_file(ds.get_name()) people = ds.get_top_people(startdate, enddate, identities_db, opts.npeople) test_json = os.path.join("json",fpeople) self.assertTrue(self._compare_data(people, test_json)) for upeople_id in people : evol_data = ds.get_person_evol(upeople_id, period, startdate, enddate, identities_db, type_analysis = None) fperson = ds().get_person_evol_file(upeople_id) test_json = os.path.join("json",fperson) self.assertTrue(self._compare_data(evol_data, test_json)) agg = ds.get_person_agg(upeople_id, startdate, enddate, identities_db, type_analysis = None) fperson = ds().get_person_agg_file(upeople_id) test_json = os.path.join("json",fperson) self.assertTrue(self._compare_data(agg, test_json))
def test_get_filter_items(self): opts = read_options() automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): Report.connect_ds(ds) for filter_ in Report.get_filters(): items = ds.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: continue if (isinstance(items, dict)): items = items['name'] if not isinstance(items, (list)): items = [items] if ds.get_name() in ["scr","pullpo"] : items = [item.replace("/","_") for item in items] elif ds.get_name() == "mls": items = [item.replace("/","_").replace("<","__").replace(">","___") for item in items] fn = ds.get_name()+"-"+filter_.get_name_plural()+".json" createJSON(items, opts.destdir+"/"+ fn) test_json = os.path.join("json",fn) new_json = opts.destdir+"/"+ fn if ds.get_name() not in ["scr","pullpo"] : # scr, pullpo repos format is more complex and # is checked already in test_get_agg_evol_filters_data self.assertTrue(self.compareJSON(test_json, new_json))
def test_get_filter_item_top (self): opts = read_options() npeople = opts.npeople automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): if ds.get_name() not in ['scm','its','mls']: continue Report.connect_ds(ds) bots = ds.get_bots() for filter_ in Report.get_filters(): items = ds.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: continue if (isinstance(items, dict)): items = items['name'] if not isinstance(items, (list)): items = [items] for item in items: filter_item = Filter(filter_.get_name(), item) top = ds.get_top_data(startdate, enddate, identities_db, filter_item, npeople) if top is None or top == {}: continue test_json = os.path.join("json",filter_item.get_top_filename(ds())) self.assertTrue(self._compare_data(top, test_json))
def init(automator_file, metrics_path=None): Report._automator_file = automator_file Report._automator = read_main_conf(automator_file) Report._init_filters() Report._init_data_sources() if metrics_path is not None: Report._init_metrics(metrics_path) studies_path = metrics_path.replace("metrics", "analysis") Report._init_studies(studies_path)
def test_get_top_data (self): opts = read_options() npeople = opts.npeople automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): Report.connect_ds(ds) top = ds.get_top_data(startdate, enddate, identities_db, None, npeople) test_json = os.path.join("json",ds.get_name()+"-top.json") self.assertTrue(self._compare_data(top, test_json))
def test_create_top_report (self): opts = read_options() automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): Report.connect_ds(ds) ds.create_top_report(startdate, enddate, opts.destdir, opts.npeople, identities_db) fn = ds.get_name()+"-top.json" test_json = os.path.join("json",fn) top_json = os.path.join(opts.destdir,fn) self.assertTrue(self.compareJSON(test_json, top_json))
def off_test_get_filter_summary (self): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): if ds.get_name() not in ['scm','its','mls']: continue Report.connect_ds(ds) for filter_ in Report.get_filters(): if (filter_.get_name() == "company"): summary = ds.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10) test_json = os.path.join("json",filter_.get_summary_filename(ds)) self.assertTrue(self._compare_data(summary, test_json))
def test_get_agg_data(self): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] # Test without filters for ds in Report.get_data_sources(): logging.info(ds.get_name() + " test_get_agg_data") # Create the evolutionary data from dbs and check with test JSON Report.connect_ds(ds) ds_data = ds.get_agg_data (period, startdate, enddate, identities_db) test_json = os.path.join("json",ds().get_agg_filename()) self.assertTrue(self._compare_data(ds_data, test_json))
def test_create_agg_evol_filters_all_data(self): opts = read_options() npeople = opts.npeople destdir = opts.destdir period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] supported_all = { "scm":["people2","company","country","repository","domain","projects","company+country","company+project"], "its":["people2","company","country","repository","domain","projects","company+country","company+project"], "its_1":["people2"], "mls":["people2","company","country","repository","domain","projects"], "scr":["people2","company","country","repository","domain","projects"], "mediawiki":["people2"], "irc":["people2"], "downloads":["people2"], "qaforums":["people2"], "releases":["people2"], "pullpo":["people2"], "eventizer":[] } for ds in Report.get_data_sources(): for filter_ in Report.get_filters(): if filter_.get_name() in supported_all[ds.get_name()]: logging.info("---> Using new filter API") ds.create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db) # Static ds_json = filter_.get_static_filename_all(ds()) if ds_json in ["scr-cou-all-evolutionary.json","scr-cou-all-static.json", "scr-dom-all-evolutionary.json","scr-dom-all-static.json", "its-dom-all-static.json"]: # The order of elements change in each execution continue f_test_json = os.path.join("json", ds_json) f_report_json = os.path.join(opts.destdir, ds_json) self.assertTrue(self.compareJSON(f_test_json, f_report_json)) # Evol ds_json = filter_.get_evolutionary_filename_all(ds()) f_test_json = os.path.join("json", ds_json) f_report_json = os.path.join(opts.destdir, ds_json) self.assertTrue(self.compareJSON(f_test_json, f_report_json))
def test_get_agg_evol_filters_data(self): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] # Test for all filters for ds in Report.get_data_sources(): Report.connect_ds(ds) for filter_ in Report.get_filters(): filter_name = filter_.get_name() if filter_name in ["company+country","company+project"]: # GROUP BY only filters continue filter_name_short = filter_.get_name_short() bots = ds.get_filter_bots(filter_) items = ds.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: continue if (isinstance(items, dict)): items = items['name'] if not isinstance(items, (list)): items = [items] for item in items: filter_item = Filter(filter_.get_name(), item) item_file = item if ds.get_name() in ["its","scr","pullpo"] : item_file = item.replace("/","_") elif ds.get_name() == "mls": item_file = item.replace("/","_").replace("<","__").replace(">","___") logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "agg") agg = ds.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-static.json" test_json = os.path.join("json",fn) self.assertTrue(self._compare_data(agg, test_json)) logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "evol") evol = ds.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-evolutionary.json" test_json = os.path.join("json",fn) self.assertTrue(self._compare_data(evol, test_json))
def test_create_evolutionary_report(self): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] # Test without filters for ds in Report.get_data_sources(): # Create the evolutionary data from dbs and check with test JSON logging.info(ds.get_name() + " create_evolutionary_report") Report.connect_ds(ds) ds.create_evolutionary_report (period, startdate, enddate, opts.destdir, identities_db) ds_json = ds().get_evolutionary_filename() f_test_json = os.path.join("json", ds_json) f_report_json = os.path.join(opts.destdir, ds_json) self.assertTrue(self.compareJSON(f_test_json, f_report_json))
os.environ["LANG"] = "" os.environ["R_LIBS"] = "../../r-lib" if __name__ == '__main__': init_env() from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON from vizgrimoire.report import Report logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s') logging.info("Starting Report analysis") opts = read_options() Report.init(opts.config_file, opts.metrics_path) automator = read_main_conf(opts.config_file) if 'start_date' not in automator['r']: logging.error("start_date (yyyy-mm-dd) not found in " + opts.config_file) sys.exit() start_date = automator['r']['start_date'] if 'end_date' not in automator['r']: end_date = time.strftime('%Y-%m-%d') else: end_date = automator['r']['end_date'] if 'period' not in automator['r']: period = getPeriod("months") else: period = getPeriod(automator['r']['period']) logging.info("Period: " + period)
os.environ["R_LIBS"] = "../../r-lib" if __name__ == '__main__': init_env() from vizgrimoire.GrimoireUtils import getPeriod, read_main_conf, createJSON from vizgrimoire.report import Report logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') logging.info("Starting Report analysis") opts = read_options() Report.init(opts.config_file, opts.metrics_path) automator = read_main_conf(opts.config_file) if 'start_date' not in automator['r']: logging.error("start_date (yyyy-mm-dd) not found in " + opts.config_file) sys.exit() start_date = automator['r']['start_date'] if 'end_date' not in automator['r']: end_date = time.strftime('%Y-%m-%d') else: end_date = automator['r']['end_date'] if 'period' not in automator['r']: period = getPeriod("months") else: period = getPeriod(automator['r']['period']) logging.info("Period: " + period)