def test_get_filter_item_top (self): opts = read_options() npeople = opts.npeople automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] for ds in Report.get_data_sources(): if ds.get_name() not in ['scm','its','mls']: continue Report.connect_ds(ds) bots = ds.get_bots() for filter_ in Report.get_filters(): items = ds.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: continue if (isinstance(items, dict)): items = items['name'] if not isinstance(items, (list)): items = [items] for item in items: filter_item = Filter(filter_.get_name(), item) top = ds.get_top_data(startdate, enddate, identities_db, filter_item, npeople) if top is None or top == {}: continue test_json = os.path.join("json",filter_item.get_top_filename(ds())) self.assertTrue(self._compare_data(top, test_json))
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db): check = False # activate to debug pull_requests filter_name = filter_.get_name() if filter_name == "people2" or filter_name == "company+country": filter_all = Filter(filter_name, None) agg_all = Pullpo.get_agg_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_static_filename_all(Pullpo())) createJSON(agg_all, fn) evol_all = Pullpo.get_evolutionary_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(Pullpo())) createJSON(evol_all, fn) if check: Pullpo._check_report_all_data(evol_all, filter_, startdate, enddate, identities_db, True, period) Pullpo._check_report_all_data(agg_all, filter_, startdate, enddate, identities_db, False, period) else: logging.error(filter_name + " does not support yet group by items sql queries")
def create_filter_report_all(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db): check = False # activate to debug issues filter_name = filter_.get_name() # Change filter to GrimoireLib notation filter_name = filter_name.replace("+", MetricFilters.DELIMITER) if filter_name in ["people2","company", "country","repository","domain","project", "company"+MetricFilters.DELIMITER+"country", "company"+MetricFilters.DELIMITER+"project"] : filter_all = Filter(filter_name, None) agg_all = cls.get_agg_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_static_filename_all(cls())) createJSON(agg_all, fn) ITS.convert_all_to_single(agg_all, filter_, destdir, False, period) evol_all = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(cls())) createJSON(evol_all, fn) ITS.convert_all_to_single(evol_all, filter_, destdir, True, period) if check: cls._check_report_all_data(evol_all, filter_, startdate, enddate, identities_db, True, period) cls._check_report_all_data(agg_all, filter_, startdate, enddate, identities_db, False, period) else: logging.error(filter_name +" does not support yet group by items sql queries")
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] fn = os.path.join(destdir, filter_.get_filename(EventsDS())) createJSON(items, fn) for item in items: item_name = "'" + item + "'" logging.info(item_name) filter_item = Filter(filter_name, item) if filter_name in ("repository"): logging.warn("Filter 'repository' detected for top info") top_authors = EventsDS.get_top_data(startdate, enddate, identities_db, filter_item, npeople) logging.warn(filter_item.get_top_filename(EventsDS())) fn = os.path.join(destdir, filter_item.get_top_filename(EventsDS())) createJSON(top_authors, fn)
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = SCR.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] fn = os.path.join(destdir, filter_.get_filename(SCR())) createJSON(items, fn) for item in items : item_name = "'"+ item+ "'" logging.info (item_name) filter_item = Filter(filter_name, item) if filter_name in ("company","project","repository"): top_mergers = SCR.get_top_data(startdate, enddate, identities_db, filter_item, npeople) fn = os.path.join(destdir, filter_item.get_top_filename(SCR())) createJSON(top_mergers, fn)
def get_list(self): from vizgrimoire.data_source import DataSource from vizgrimoire.filter import Filter startdate = self.filters.startdate enddate = self.filters.enddate closed_condition = ITS._get_closed_condition() if self.filters.closed_condition is not None: closed_condition = self.filters.closed_condition bots = DataSource.get_filter_bots(Filter("domain")) fbots = '' for bot in bots: fbots += " dom.name<>'" + bot + "' and " tables = Set([]) filters = Set([]) tables.union_update(self.db.GetTablesDomains(self.db.identities_db)) tables_str = self.db._get_tables_query(tables) filters.union_update(self.db.GetFiltersDomains()) filters_str = self.db._get_filters_query(filters) q = "SELECT DISTINCT(SUBSTR(email,LOCATE('@',email)+1)) as domain "+\ "FROM "+ tables_str + " "+\ "WHERE " + filters_str +" AND "+\ " "+ fbots +" "+\ " c.changed_on >= "+ startdate+ " AND "+\ " c.changed_on < "+ enddate+ " AND "+\ " "+ closed_condition+" "+\ "GROUP BY domain "+\ "ORDER BY COUNT(DISTINCT(c.issue_id)) DESC, domain LIMIT " + str(Metrics.domains_limit) data = self.db.ExecuteQuery(q) data['name'] = data.pop('domain') return (data)
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db): check = False # activate to debug issues filter_name = filter_.get_name() if filter_name in ["people2","company","repository","country","domain","project"] : filter_all = Filter(filter_name, None) agg_all = SCR.get_agg_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_static_filename_all(SCR())) createJSON(agg_all, fn) SCR.convert_all_to_single(agg_all, filter_, destdir, False, period) evol_all = SCR.get_evolutionary_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(SCR())) createJSON(evol_all, fn) SCR.convert_all_to_single(evol_all, filter_, destdir, True, period) if check: SCR._check_report_all_data(evol_all, filter_, startdate, enddate, identities_db, True, period) SCR._check_report_all_data(agg_all, filter_, startdate, enddate, identities_db, False, period) else: logging.error(filter_name +" does not support yet group by items sql queries")
def create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = MLS.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] for item in items: item = item.replace("'", "\\'") item_name = "'" + item + "'" logging.info(item_name) filter_item = Filter(filter_.get_name(), item) top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False) createJSON(top_senders, destdir + "/" + filter_item.get_top_filename(MLS()))
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = IRC.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return if not isinstance(items, (list)): items = [items] fn = os.path.join(destdir, filter_.get_filename(IRC())) createJSON(items, fn) for item in items: # item_name = "'"+ item+ "'" logging.info(item) filter_item = Filter(filter_.get_name(), item) evol_data = IRC.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_evolutionary_filename(IRC())) createJSON( completePeriodIds(evol_data, period, startdate, enddate), fn) agg = IRC.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(IRC())) createJSON(agg, fn)
def create_filter_report(cls, filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = cls.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] fn = os.path.join(destdir, filter_.get_filename(cls())) createJSON(items, fn) if filter_name in ("domain", "company", "repository"): items_list = {'name' : [], 'closed_365' : [], 'closers_365' : []} else: items_list = items for item in items : item_name = "'"+ item+ "'" logging.info (item_name) filter_item = Filter(filter_name, item) evol_data = cls.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls())) createJSON(evol_data, fn) agg = cls.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(cls())) createJSON(agg, fn) if filter_name in ["domain", "company", "repository"]: items_list['name'].append(item.replace('/', '_')) items_list['closed_365'].append(agg['closed_365']) items_list['closers_365'].append(agg['closers_365']) if filter_name in ["company","domain","repository"]: top = cls.get_top_data(startdate, enddate, identities_db, filter_item, npeople) fn = os.path.join(destdir, filter_item.get_top_filename(cls())) createJSON(top, fn) fn = os.path.join(destdir, filter_.get_filename(cls())) createJSON(items_list, fn) if (filter_name == "company"): ds = ITS #summary = cls.get_filter_summary( # filter_, period, startdate, enddate, # identities_db, 10 # ) #createJSON (summary, # destdir + "/" + filter_.get_summary_filename(cls)) # Perform ages study, if it is specified in Report cls.ages_study_com (items, period, startdate, enddate, destdir)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = EventsDS.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return filter_name = filter_.get_name() items = items['name'] if not isinstance(items, list): items = [items] file_items = [] for item in items: if re.compile("^\..*").match(item) is not None: item = "_" + item file_items.append(item) fn = os.path.join(destdir, filter_.get_filename(EventsDS())) createJSON(file_items, fn) if filter_name in ("repository"): items_list = {'name': [], 'events_365': [], 'rsvps_365': []} else: items_list = items for item in items: logging.info(item) filter_item = Filter(filter_.get_name(), item) evol_data = EventsDS.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join( destdir, filter_item.get_evolutionary_filename(EventsDS())) createJSON( completePeriodIds(evol_data, period, startdate, enddate), fn) agg = EventsDS.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(EventsDS())) createJSON(agg, fn) if filter_name in ("repository"): items_list['name'].append(item.replace('/', '_')) items_list['events_365'].append(agg['events_365']) items_list['rsvps_365'].append(agg['rsvps_365']) EventsDS.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db) fn = os.path.join(destdir, filter_.get_filename(EventsDS())) createJSON(items_list, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = SCR.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] # Include metrics to sort in javascript. items_list = { "name": [], "review_time_days_median": [], "submitted": [] } for item in items: item_file = item.replace("/", "_") items_list["name"].append(item_file) logging.info(item) filter_item = Filter(filter_name, item) evol = SCR.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCR())) createJSON(evol, fn) # Static agg = SCR.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(SCR())) createJSON(agg, fn) if 'submitted' in agg: items_list["submitted"].append(agg["submitted"]) else: items_list["submitted"].append("NA") if 'review_time_days_median' in agg: items_list["review_time_days_median"].append( agg['review_time_days_median']) else: items_list["review_time_days_median"].append("NA") fn = os.path.join(destdir, filter_.get_filename(SCR())) createJSON(items_list, fn) SCR.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db)
def ages_study_com(ds, items, period, startdate, enddate, destdir): """Perform ages study for companies, if it is specified in Report. Produces JSON files for those studies. Parameters ---------- ds: { SCM | ITS | MLS } Data source items: ?? Items period: ?? Period startdate: ?? Start date enddate: ?? End date destdir: string Directory for writing the JSON files """ from vizgrimoire.report import Report filter_name = "company" studies = Report.get_studies() ages = None for study in studies: if study.id == "ages": ages = study if ages is not None: # Get config parameters for producing a connection # to the database config = Report.get_config() db_identities = config['generic']['db_identities'] dbuser = config['generic']['db_user'] dbpass = config['generic']['db_password'] start_string = ds.get_name() + "_start_date" end_string = ds.get_name() + "_end_date" if start_string in config['r']: startdate = "'" + config['r'][start_string] + "'" if end_string in config['r']: enddate = "'" + config['r'][end_string] + "'" ds_dbname = ds.get_db_name() dbname = config['generic'][ds_dbname] dsquery = ds.get_query_builder() dbcon = dsquery(dbuser, dbpass, dbname, db_identities) for item in items: filter_item = Filter(filter_name, item) metric_filters = MetricFilters(period, startdate, enddate, filter_item.get_type_analysis()) obj = ages(dbcon, metric_filters) res = obj.create_report(ds, destdir)
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db): # New API for getting all metrics with one query check = False # activate to debug issues filter_name = filter_.get_name() # Change filter to GrimoireLib notation filter_name = filter_name.replace("+", MetricFilters.DELIMITER) # This report is created per item, not using GROUP BY yet SCM.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db) # Filters metrics computed using GROUP BY queries if filter_name in [ "people2", "company", "country", "repository", "domain", "project", "company" + MetricFilters.DELIMITER + "country", "company" + MetricFilters.DELIMITER + "project" ]: filter_all = Filter(filter_name, None) agg_all = SCM.get_agg_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_static_filename_all(SCM())) createJSON(agg_all, fn) SCM.convert_all_to_single(agg_all, filter_, destdir, False, period) evol_all = SCM.get_evolutionary_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(SCM())) createJSON(evol_all, fn) SCM.convert_all_to_single(evol_all, filter_, destdir, True, period) # Studies report for filters if (filter_name == "company"): ds = SCM #summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10) #createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM)) # Perform ages study, if it is specified in Report SCM.ages_study_com(agg_all['name'], period, startdate, enddate, destdir) # check is only done fpr basic filters. Composed should work if basic does. if check and not "," in filter_name: SCM._check_report_all_data(evol_all, filter_, startdate, enddate, identities_db, True, period) SCM._check_report_all_data(agg_all, filter_, startdate, enddate, identities_db, False, period) else: logging.error(filter_name + " does not support yet group by items sql queries")
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = SCM.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] fn = os.path.join(destdir, filter_.get_filename(SCM())) escaped_items = [i.replace('/', '_') for i in items] logging.info("D1") createJSON(escaped_items, fn) for item in items: item_name = "'" + item + "'" logging.info(item_name) filter_item = Filter(filter_name, item) evol_data = SCM.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_evolutionary_filename(SCM())) createJSON(evol_data, fn) agg = SCM.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(SCM())) createJSON(agg, fn) SCM.create_filter_report_top(filter_, period, startdate, enddate, destdir, npeople, identities_db) if (filter_name == "company"): ds = SCM #summary = SCM.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10) #createJSON (summary, destdir+"/"+ filter_.get_summary_filename(SCM)) # Perform ages study, if it is specified in Report SCM.ages_study_com(items, period, startdate, enddate, destdir)
def create_filter_report_all(filter_, period, startdate, enddate, destdir, npeople, identities_db): filter_name = filter_.get_name() if filter_name == "people2" or filter_name == "company_off": filter_all = Filter(filter_name, None) agg_all = IRC.get_agg_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_static_filename_all(IRC())) createJSON(agg_all, fn) evol_all = IRC.get_evolutionary_data(period, startdate, enddate, identities_db, filter_all) fn = os.path.join(destdir, filter_.get_evolutionary_filename_all(IRC())) createJSON(evol_all, fn) else: logging.error(IRC.get_name() + " " + filter_name + " does not support yet group by items sql queries")
def test_get_agg_evol_filters_data(self): opts = read_options() period = getPeriod(opts.granularity) automator = read_main_conf(opts.config_file) identities_db = automator['generic']['db_identities'] # Test for all filters for ds in Report.get_data_sources(): Report.connect_ds(ds) for filter_ in Report.get_filters(): filter_name = filter_.get_name() if filter_name in ["company+country","company+project"]: # GROUP BY only filters continue filter_name_short = filter_.get_name_short() bots = ds.get_filter_bots(filter_) items = ds.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: continue if (isinstance(items, dict)): items = items['name'] if not isinstance(items, (list)): items = [items] for item in items: filter_item = Filter(filter_.get_name(), item) item_file = item if ds.get_name() in ["its","scr","pullpo"] : item_file = item.replace("/","_") elif ds.get_name() == "mls": item_file = item.replace("/","_").replace("<","__").replace(">","___") logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "agg") agg = ds.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-static.json" test_json = os.path.join("json",fn) self.assertTrue(self._compare_data(agg, test_json)) logging.info(ds.get_name() +","+ filter_name+","+ item+","+ "evol") evol = ds.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = item_file+"-"+ds.get_name()+"-"+filter_name_short+"-evolutionary.json" test_json = os.path.join("json",fn) self.assertTrue(self._compare_data(evol, test_json))
def get_list(self): filter_ = DataSource.get_filter_bots(Filter("country")) filter_countries = '' for country in filter_: filter_countries += " cou.name<>'" + country + "' AND " countries_tables = self.db._get_tables_query( self.db.GetTablesCountries()) countries_filters = self.db._get_filters_query( self.db.GetFiltersCountries()) q = "SELECT cou.name as name, COUNT(m.message_ID) as sent "+\ "FROM "+ countries_tables + " "+\ "WHERE "+ countries_filters + " AND "+\ " "+ filter_countries+ " "+\ " m.first_date >= "+self.filters.startdate+" AND "+\ " m.first_date < "+self.filters.enddate+" "+\ "GROUP BY cou.name "+\ "ORDER BY COUNT((m.message_ID)) DESC, cou.name " data = self.db.ExecuteQuery(q) return (data['name'])
def _check_report_all_data(data, filter_, startdate, enddate, idb, evol=False, period=None): # Check per item data with group by people data items = SCM.get_filter_items(filter_, startdate, enddate, idb) id_field = DSQuery.get_group_field(filter_.get_name()) id_field = id_field.split('.')[1] # remove table name for i in range(0, len(items['name'])): name = items['name'][i] logging.info("Checking " + name + " " + str(i) + "/" + str(len(items['name']))) if filter_.get_name() == "people2": uuid = items['id'][i] item = uuid else: item = name pos = data[id_field].index(name) type_analysis = [filter_.get_name(), item] filter_item = Filter(filter_.get_name(), item) if not evol: if filter_.get_name() == "people2": agg = SCM.get_person_agg(uuid, startdate, enddate, idb, type_analysis) else: agg = SCM.get_agg_data(period, startdate, enddate, idb, filter_item) assert agg['commits'] == data['commits'][pos] else: if filter_.get_name() == "people2": ts = SCM.get_person_evol(uuid, period, startdate, enddate, idb, type_analysis) else: ts = SCM.get_evolutionary_data(period, startdate, enddate, idb, filter_item) assert ts['commits'] == data['commits'][pos]
def get_list(self): filter_ = DataSource.get_filter_bots(Filter("company")) filter_organizations = '' for company in filter_: filter_organizations += " org.name<>'" + company + "' AND " organizations_tables = self.db._get_tables_query( self.db.GetTablesCompanies()) organizations_filters = self.db._get_filters_query( self.db.GetFiltersCompanies()) q = "SELECT org.name as name, COUNT(DISTINCT(m.message_ID)) as sent "+\ " FROM "+ organizations_tables + " "+\ " WHERE "+ organizations_filters + " AND "+\ " "+ filter_organizations+ " "+\ " m.first_date >= "+self.filters.startdate+" AND "+\ " m.first_date < "+self.filters.enddate+" "+\ " GROUP BY org.name "+\ " ORDER BY COUNT(DISTINCT(m.message_ID)) DESC" data = self.db.ExecuteQuery(q) return (data['name'])
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = QAForums.get_filter_items(filter_, startdate, enddate, identities_db) if items == None: return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, list): items = [items] file_items = [] for item in items: if re.compile("^\..*").match(item) is not None: item = "_" + item file_items.append(item) fn = os.path.join(destdir, filter_.get_filename(QAForums())) createJSON(file_items, fn) for item in items: logging.info(item) filter_item = Filter(filter_.get_name(), item) evol_data = QAForums.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join( destdir, filter_item.get_evolutionary_filename(QAForums())) createJSON( completePeriodIds(evol_data, period, startdate, enddate), fn) agg = QAForums.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(QAForums())) createJSON(agg, fn)
def get_list(self): from vizgrimoire.data_source import DataSource from vizgrimoire.filter import Filter bots = DataSource.get_filter_bots(Filter("company")) fbots = '' for bot in bots: fbots += " org.name<>'"+bot+"' and " startdate = self.filters.startdate enddate = self.filters.enddate closed_condition = ITS._get_closed_condition() if self.filters.closed_condition is not None: closed_condition = self.filters.closed_condition # list each of the organizations analyzed # those are order by number of closed issues q = "select org.name "+\ "from issues i, "+\ " changes ch, "+\ " people_uidentities pup, "+\ " "+ self.db.identities_db+ ".enrollments enr, "+\ " "+ self.db.identities_db+ ".organizations org "+\ "where i.id = ch.issue_id and "+\ " ch.changed_by = pup.people_id and "+\ " pup.uuid = enr.uuid and "+\ " enr.organization_id = org.id and "+\ " ch.changed_on >= "+ startdate+ " and "+\ " ch.changed_on < "+ enddate+" and "+\ " i.submitted_on >= enr.start and "+\ " i.submitted_on < enr.end and "+\ " "+ fbots +\ closed_condition +\ " group by org.name "+\ " order by count(distinct(i.id)) desc" data = self.db.ExecuteQuery(q) return (data)
def convert_all_to_single(cls, data, filter_, destdir, evolutionary, period='month'): """ Convert a GROUP BY result to follow tradition individual JSON files """ from vizgrimoire.SCM import SCM from vizgrimoire.ITS import ITS from vizgrimoire.SCR import SCR from vizgrimoire.MLS import MLS from vizgrimoire.filter import Filter if cls == ITS or cls == SCR: if 'url' in data.keys(): data['name'] = data.pop('url') data['name'] = [ item.replace('/', '_') for item in data['name'] ] elif cls == MLS: if 'mailing_list_url' in data.keys(): data['name'] = data.pop('mailing_list_url') data['name'] = [ item.replace('/', '_') for item in data['name'] ] if cls == ITS or cls == SCM: if filter_.get_name() in ["company+country", "company+project"]: data['name'] = data.pop('filter') if not evolutionary: # First create the JSON with the list of items item_list = {} fn = os.path.join(destdir, filter_.get_filename(cls)) if cls == SCM: fields = ["authors_365", "name", "commits_365"] elif cls == ITS: fields = ["closed_365", "closers_365", "name"] elif cls == MLS: fields = ["sent_365", "senders_365", "name"] elif cls == SCR: fields = [ "submitted", "review_time_days_median", "review_time_pending_upload_ReviewsWaitingForReviewer_days_median", "name" ] else: fields = ["name"] for field in fields: if field in data: item_list[field] = data[field] createJSON(item_list, fn) # Items files ts_fields = ['unixtime', 'id', 'date', period] # Not metrics fields no_metrics_fields = ['filter_type'] for i in range(0, len(data['name'])): item_metrics = {} item = data['name'][i] for metric in data: if metric == "name": continue if metric in ts_fields + no_metrics_fields: continue if len(data[metric]) < len(data['name']): logging.error(cls.get_name() + " " + metric + " not supported in GROUP BY. Not included") continue if evolutionary: if not isinstance(data[metric][i], list): logging.error( cls.get_name() + " " + metric + " evol not supported in GROUP BY. Not included") continue item_metrics[metric] = data[metric][i] filter_item = Filter(filter_.get_name(), item) if evolutionary: for field in ts_fields: # Shared time series fields item_metrics[field] = data[field] fn = os.path.join(destdir, filter_item.get_evolutionary_filename(cls())) else: fn = os.path.join(destdir, filter_item.get_static_filename(cls())) createJSON(item_metrics, fn)
def create_filter_report(filter_, period, startdate, enddate, destdir, npeople, identities_db): from vizgrimoire.report import Report items = Report.get_items() if items is None: items = MLS.get_filter_items(filter_, startdate, enddate, identities_db) if (items == None): return items = items['name'] filter_name = filter_.get_name() if not isinstance(items, (list)): items = [items] items_files = [ item.replace('/', '_').replace("<", "__").replace(">", "___") for item in items ] fn = os.path.join(destdir, filter_.get_filename(MLS())) createJSON(items_files, fn) if filter_name in ("domain", "company", "repository"): items_list = {'name': [], 'sent_365': [], 'senders_365': []} else: items_list = items for item in items: item = item.replace("'", "\\'") item_name = "'" + item + "'" logging.info(item_name) filter_item = Filter(filter_.get_name(), item) evol_data = MLS.get_evolutionary_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_evolutionary_filename(MLS())) createJSON(evol_data, fn) agg = MLS.get_agg_data(period, startdate, enddate, identities_db, filter_item) fn = os.path.join(destdir, filter_item.get_static_filename(MLS())) createJSON(agg, fn) if filter_name in ("domain", "company", "repository"): items_list['name'].append( item.replace('/', '_').replace("<", "__").replace(">", "___")) items_list['sent_365'].append(agg['sent_365']) items_list['senders_365'].append(agg['senders_365']) top_senders = MLS.get_top_data(startdate, enddate, identities_db, filter_item, npeople, False) createJSON(top_senders, destdir + "/" + filter_item.get_top_filename(MLS())) fn = os.path.join(destdir, filter_.get_filename(MLS())) createJSON(items_list, fn) if (filter_name == "company"): ds = MLS if False: summary = MLS.get_filter_summary(filter_, period, startdate, enddate, identities_db, 10, self.db.projects_db) createJSON(summary, destdir + "/" + filter_.get_summary_filename(MLS)) # Perform ages study, if it is specified in Report MLS.ages_study_com(items, period, startdate, enddate, destdir)