def get_top_data_authors(startdate, enddate, i_db, filter_, npeople): top = {} mauthors = DataSource.get_metrics("authors", SCM) if mauthors is None: return top period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) mfilter.global_filter = mauthors.filters.global_filter if filter_ is None: top['authors.'] = mauthors.get_list(mfilter, 0) top['authors.last month'] = mauthors.get_list(mfilter, 31) top['authors.last year'] = mauthors.get_list(mfilter, 365) elif filter_.get_name() in ["company", "repository", "project"]: if filter_.get_name() in ["company", "repository", "project"]: top['authors.'] = mauthors.get_list(mfilter, 0) top['authors.last month'] = mauthors.get_list(mfilter, 31) top['authors.last year'] = mauthors.get_list(mfilter, 365) else: # If we have performance issues with tops, remove filters above # to avoid computing trends for tops top = mauthors.get_list(mfilter) else: logging.info("Top authors not support for " + filter_.get_name()) return top
def get_trends(self, date, days): """ Returns the trend metrics between now and now-days values """ if self.filters.type_analysis and self.filters.type_analysis[1] is None: return self._get_trends_all_items(date, days) # Keeping state of origin filters filters = self.filters chardates = GetDates(date, days) self.filters = MetricFilters(filters.period, chardates[1], chardates[0], filters.type_analysis) self.filters.global_filter = filters.global_filter self.filters.closed_condition = filters.closed_condition last = self.get_agg() if last[self.id] is not None: last = int(last[self.id]) else: last = 0 self.filters = MetricFilters(filters.period, chardates[2], chardates[1], filters.type_analysis) self.filters.global_filter = filters.global_filter self.filters.closed_condition = filters.closed_condition prev = self.get_agg() if prev[self.id] is not None: prev = int(prev[self.id]) else: prev = 0 data = {} data['diff_net'+self.id+'_'+str(days)] = last - prev data['percentage_'+self.id+'_'+str(days)] = GetPercentageDiff(prev, last) data[self.id+'_'+str(days)] = last # Returning filters to their original value self.filters = filters return (data)
def get_top_data_authors (startdate, enddate, i_db, filter_, npeople): top = {} mauthors = DataSource.get_metrics("authors", SCM) if mauthors is None: return top period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) mfilter.global_filter = mauthors.filters.global_filter if filter_ is None: top['authors.'] = mauthors.get_list(mfilter, 0) top['authors.last month'] = mauthors.get_list(mfilter, 31) top['authors.last year'] = mauthors.get_list(mfilter, 365) elif filter_.get_name() in ["company","repository","project"]: if filter_.get_name() in ["company","repository","project"]: top['authors.'] = mauthors.get_list(mfilter, 0) top['authors.last month'] = mauthors.get_list(mfilter, 31) top['authors.last year'] = mauthors.get_list(mfilter, 365) else: # If we have performance issues with tops, remove filters above # to avoid computing trends for tops top = mauthors.get_list(mfilter) else: logging.info("Top authors not support for " + filter_.get_name()) return top
def test_scm_simple(self): correct = """ {'aging': ActorsDuration: {'age': datetime.timedelta(13, 3349), 'id': u'6872b4eeab3d91e5f47f142137b6672f64dc5102', 'name': u'6872b4eeab3d91e5f47f142137b6672f64dc5102'} {'age': datetime.timedelta(30, 14598), 'id': u'86600750577bd979bcf1744a5d8aa387e3c0e088', 'name': u'86600750577bd979bcf1744a5d8aa387e3c0e088'} {'age': datetime.timedelta(16, 1472), 'id': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1', 'name': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1'} {'age': datetime.timedelta(29, 12176), 'id': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e', 'name': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e'} , 'birth': ActorsDuration: {'age': datetime.timedelta(13, 3349), 'id': u'6872b4eeab3d91e5f47f142137b6672f64dc5102', 'name': u'6872b4eeab3d91e5f47f142137b6672f64dc5102'} {'age': datetime.timedelta(30, 14598), 'id': u'86600750577bd979bcf1744a5d8aa387e3c0e088', 'name': u'86600750577bd979bcf1744a5d8aa387e3c0e088'} {'age': datetime.timedelta(16, 1472), 'id': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1', 'name': u'9d9c8d35f9a5e8537debda180d0d5a818ed453c1'} {'age': datetime.timedelta(29, 12176), 'id': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e', 'name': u'c2c10e1bff19bb5783c0eea4beb9cf73df3e314e'} } """ correct = ''.join(correct.split()) filters = MetricFilters("months", "'2013-12-01'", "'2014-01-01'", []) dbcon = DSQuery(user="******", password="******", database=db_cvsanaly, identities_db=db_identities) ages = Ages(dbcon, filters) result = ages.result(SCM) self.assertEqual(''.join(str(result).split()), correct)
def releases_info(startdate, enddate, project, opts, people_out, affs_out): # Releases information. data = {} filters = MetricFilters("month", startdate, enddate, ["project", str(project)], opts.npeople, people_out, affs_out) # SCM report scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities, opts.dbprojects) dataset = scm_report(scm_dbcon, filters) data["scm"] = dataset #ITS report its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities, opts.dbprojects) dataset = its_report(its_dbcon, filters) data["its"] = dataset #SCR Report scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities, opts.dbprojects) dataset = scr_report(scr_dbcon, filters) data["scr"] = dataset return data
def integrated_projects_activity(dbcon, opts, people_out, affs_out): # Commits per integrated project projects = integrated_projects(dbcon) projects_ids = projects["subproject_id"] projects_list = [] commits_list = [] period = "month" releases = opts.releases.split(",")[-2:] startdate = "'" + releases[0] + "'" enddate = "'" + releases[1] + "'" for project_id in projects_ids: project_title = "'" + project_id + "'" type_analysis = ["project", project_title] project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, affs_out) commits = scm.Commits(dbcon, project_filters) projects_list.append(project_id) commits_list.append(int(commits.get_agg()["commits"])) createCSV({ "projects": projects_list, "commits": commits_list }, "./release/integrated_projects_commits.csv")
def integrated_projects_top_contributors(scm_dbcon, people_out, affs_out): # Top contributor per integrated project projects = integrated_projects(scm_dbcon) projects_ids = projects["subproject_id"] projects_list = [] commits = [] top_contributors = [] period = "month" releases = opts.releases.split(",")[-2:] startdate = "'" + releases[0] + "'" enddate = "'" + releases[1] + "'" for project_id in projects_ids: project_title = "'" + project_id + "'" type_analysis = ["project", project_title] project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, affs_out) authors = scm.Authors(scm_dbcon, project_filters) activity = authors.get_list() projects_list.append(project_id) commits.append(activity["commits"][0]) top_contributors.append(activity["authors"][0]) createCSV( { "projects": projects_list, "commits": commits, "contributors": top_contributors }, "./release/integrated_projects_top_contributors.csv")
def create_reports_studies(period, startdate, enddate, destdir): from vizgrimoire.metrics.metrics_filter import MetricFilters db_identities = Report.get_config()['generic']['db_identities'] dbuser = Report.get_config()['generic']['db_user'] dbpass = Report.get_config()['generic']['db_password'] studies = Report.get_studies() metric_filters = MetricFilters(period, startdate, enddate, []) for ds in get_enabled_data_sources(): ds_dbname = ds.get_db_name() dbname = Report.get_config()['generic'][ds_dbname] dsquery = ds.get_query_builder() dbcon = dsquery(dbuser, dbpass, dbname, db_identities) # logging.info(ds.get_name() + " studies active " + str(studies)) for study in studies: logging.info("Creating report for " + study.id + " for " + ds.get_name()) try: obj = study(dbcon, metric_filters) obj.create_report(ds, destdir) except TypeError: import traceback logging.info(study.id + " does no support standard API. Not used.") traceback.print_exc(file=sys.stdout) continue
def integrated_projects_top_orgs(dbcon, people_out, affs_out): # Top orgs contributing to each integrated project # The two first companies are depicted. projects = integrated_projects(dbcon) projects_ids = projects["subproject_id"] projects_list = [] commits_top1 = [] commits_top2 = [] orgs_top1 = [] orgs_top2 = [] period = "month" releases = opts.releases.split(",")[-2:] startdate = "'"+releases[0]+"'" enddate = "'"+releases[1]+"'" for project_id in projects_ids: project_title = "'" + project_id + "'" type_analysis = ["project", project_title] project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, affs_out) companies = scm.Companies(dbcon, project_filters) activity = companies.get_list() projects_list.append(project_id) commits_top1.append(activity["company_commits"][0]) commits_top2.append(activity["company_commits"][1]) orgs_top1.append(activity["name"][0]) orgs_top2.append(activity["name"][1]) createCSV({"projects":projects_list, "commitstopone":commits_top1, "commitstoptwo":commits_top2, "orgstopone":orgs_top1, "orgstoptwo":orgs_top2}, "./release/integrated_projects_top_orgs.csv")
def timezone_analysis(opts): from vizgrimoire.analysis.timezone import Timezone from vizgrimoire.SCM import SCM from vizgrimoire.MLS import MLS scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities, opts.dbprojects) mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats, opts.dbidentities, opts.dbprojects) period = "month" releases = opts.releases.split(",")[-2:] startdate = "'" + releases[0] + "'" enddate = "'" + releases[1] + "'" filters = MetricFilters(period, startdate, enddate, [], 10, "", "") tz = Timezone(scm_dbcon, filters) dataset = tz.result(SCM) labels = dataset["tz"] commits = dataset["commits"] authors = dataset["authors"] bar_chart("Timezone git activity", labels, commits, "commits_tz", authors, ["commits", "authors"]) tz = Timezone(mls_dbcon, filters) dataset = tz.result(MLS) messages = dataset["messages"] authors = dataset["authors"] bar_chart("Timezone mailing list activity", labels, messages, "messages_tz", authors, ["messages", "authors"])
def get_top_people(startdate, enddate, idb): """Top people for all data sources.""" import vizgrimoire.GrimoireSQL from vizgrimoire.SCR import SCR from vizgrimoire.MLS import MLS from vizgrimoire.ITS import ITS from vizgrimoire.IRC import IRC from vizgrimoire.Mediawiki import Mediawiki from vizgrimoire.metrics.metrics_filter import MetricFilters from vizgrimoire.data_source import DataSource npeople = "10000" # max limit, all people included min_data_sources = 3 # min data sources to be in the list tops = {} all_top = {} all_top_min_ds = {} period = None type_analysis = None mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) # SCR and SCM are the same. Don't use both for Tops mopeners = DataSource.get_metrics("submitters", SCR) if mopeners: tops["scr"] = mopeners.get_list(mfilter, 0) tops["scr"]["identifier"] = tops["scr"].pop("openers") msenders = DataSource.get_metrics("senders", MLS) if msenders: tops["mls"] = msenders.get_list(mfilter, 0) tops["mls"]["identifier"] = tops["mls"].pop("senders") mopeners = DataSource.get_metrics("openers", ITS) if mopeners: tops["its"] = mopeners.get_list(mfilter, 0) tops["its"]["identifier"] = tops["its"].pop("openers") msenders = DataSource.get_metrics("senders", IRC) if msenders: tops["irc"] = msenders.get_list(mfilter, 0) tops["irc"]["identifier"] = tops["irc"].pop("senders") mauthors = DataSource.get_metrics("authors", Mediawiki) if mauthors: tops["mediawiki"] = mauthors.get_list(mfilter, 0) tops["mediawiki"]["identifier"] = tops["mediawiki"].pop("reviews") # Build the consolidated top list using all data sources data # Only people in all data sources is used for ds in tops: pos = 1 for id in tops[ds]['id']: if id not in all_top: all_top[id] = [] all_top[id].append({ "ds": ds, "pos": pos, "identifier": tops[ds]['identifier'][pos - 1] }) pos += 1 for id in all_top: if len(all_top[id]) >= min_data_sources: all_top_min_ds[id] = all_top[id] return all_top_min_ds
def get_top_data (startdate, enddate, identities_db, filter_, npeople): bots = SCR.get_bots() top_all = None mreviewers = DataSource.get_metrics("reviewers", SCR) mopeners = DataSource.get_metrics("submitters", SCR) mmergers = DataSource.get_metrics("closers", SCR) mcorereviewers = DataSource.get_metrics("active_core_reviewers", SCR) mparticipants = DataSource.get_metrics("participants", SCR) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is None: top_reviewers = {} top_reviewers['reviewers'] = mreviewers.get_list(mfilter, 0) top_reviewers['reviewers.last month']= mreviewers.get_list(mfilter, 31) top_reviewers['reviewers.last year']= mreviewers.get_list(mfilter, 365) top_openers = {} top_openers['openers.'] = mopeners.get_list(mfilter, 0) top_openers['openers.last month']= mopeners.get_list(mfilter, 31) top_openers['openers.last year'] = mopeners.get_list(mfilter, 365) top_mergers = {} top_mergers['mergers.'] = mmergers.get_list(mfilter, 0) top_mergers['mergers.last month'] = mmergers.get_list(mfilter, 31) top_mergers['mergers.last year'] = mmergers.get_list(mfilter, 365) top_participants = {} top_participants['participants.'] = mparticipants.get_list(mfilter, 0) top_participants['participants.last month'] = mparticipants.get_list(mfilter, 31) top_participants['participants.last year'] = mparticipants.get_list(mfilter, 365) top_core_reviewers = {} top_core_reviewers['active_core_reviewers.'] = mcorereviewers.get_list(mfilter, 0) top_core_reviewers['active_core_reviewers.last month'] = mcorereviewers.get_list(mfilter, 31) top_core_reviewers['active_core_reviewers.last year'] = mcorereviewers.get_list(mfilter, 365) # The order of the list item change so we can not check it top_all = dict(top_reviewers.items() + top_openers.items() + top_mergers.items() + top_participants.items() + top_core_reviewers.items()) else: filter_name = filter_.get_name() if filter_name in ["company","domain","repository"]: if filter_name in ["company","domain","repository"]: top_all = {} top_all['active_core_reviewers.'] = mcorereviewers.get_list(mfilter, 0) top_all['active_core_reviewers.last month'] = mcorereviewers.get_list(mfilter, 31) top_all['active_core_reviewers.last year'] = mcorereviewers.get_list(mfilter, 365) else: # Remove filters above if there are performance issues top_all = mcorereviewers.get_list(mfilter) else: top_all = None return (top_all)
def get_top_data(startdate, enddate, identities_db, filter_, npeople): bots = Pullpo.get_bots() top_all = None mreviewers = DataSource.get_metrics("reviewers", Pullpo) msubmitters = DataSource.get_metrics("submitters", Pullpo) mclosers = DataSource.get_metrics("closers", Pullpo) mmergers = DataSource.get_metrics("mergers", Pullpo) mparticipants = DataSource.get_metrics("participants", Pullpo) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is None: top_reviewers = {} top_reviewers['reviewers'] = mreviewers.get_list(mfilter, 0) top_reviewers['reviewers.last month'] = mreviewers.get_list( mfilter, 31) top_reviewers['reviewers.last year'] = mreviewers.get_list( mfilter, 365) top_submitters = {} top_submitters['submitters.'] = msubmitters.get_list(mfilter, 0) top_submitters['submitters.last month'] = msubmitters.get_list( mfilter, 31) top_submitters['submitters.last year'] = msubmitters.get_list( mfilter, 365) top_closers = {} top_closers['closers.'] = mclosers.get_list(mfilter, 0) top_closers['closers.last month'] = mclosers.get_list(mfilter, 31) top_closers['closers.last year'] = mclosers.get_list(mfilter, 365) top_mergers = {} top_mergers['mergers.'] = mmergers.get_list(mfilter, 0) top_mergers['mergers.last month'] = mmergers.get_list(mfilter, 31) top_mergers['mergers.last year'] = mmergers.get_list(mfilter, 365) top_participants = {} top_participants['participants.'] = mparticipants.get_list( mfilter, 0) top_participants[ 'participants.last month'] = mparticipants.get_list( mfilter, 31) top_participants[ 'participants.last year'] = mparticipants.get_list( mfilter, 365) # The order of the list item change so we can not check it top_all = dict(top_reviewers.items() + top_submitters.items() + top_closers.items() + top_mergers.items() + top_participants.items()) else: logging.info("Pullpo does not support yet top for filters.") return (top_all)
def ages_study_com(ds, items, period, startdate, enddate, destdir): """Perform ages study for companies, if it is specified in Report. Produces JSON files for those studies. Parameters ---------- ds: { SCM | ITS | MLS } Data source items: ?? Items period: ?? Period startdate: ?? Start date enddate: ?? End date destdir: string Directory for writing the JSON files """ from vizgrimoire.report import Report filter_name = "company" studies = Report.get_studies() ages = None for study in studies: if study.id == "ages": ages = study if ages is not None: # Get config parameters for producing a connection # to the database config = Report.get_config() db_identities = config['generic']['db_identities'] dbuser = config['generic']['db_user'] dbpass = config['generic']['db_password'] start_string = ds.get_name() + "_start_date" end_string = ds.get_name() + "_end_date" if start_string in config['r']: startdate = "'" + config['r'][start_string] + "'" if end_string in config['r']: enddate = "'" + config['r'][end_string] + "'" ds_dbname = ds.get_db_name() dbname = config['generic'][ds_dbname] dsquery = ds.get_query_builder() dbcon = dsquery(dbuser, dbpass, dbname, db_identities) for item in items: filter_item = Filter(filter_name, item) metric_filters = MetricFilters(period, startdate, enddate, filter_item.get_type_analysis()) obj = ages(dbcon, metric_filters) res = obj.create_report(ds, destdir)
def get_top_data(startdate, enddate, identities_db, filter_, npeople, threads_top=True): msenders = DataSource.get_metrics("senders", MLS) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) top = {} if filter_ is None: top['senders.'] = msenders.get_list(mfilter, 0) top['senders.last month'] = msenders.get_list(mfilter, 31) top['senders.last year'] = msenders.get_list(mfilter, 365) if threads_top: top['threads.'] = MLS.getLongestThreads( startdate, enddate, identities_db, npeople) startdate = datetime.date.today() - datetime.timedelta( days=365) startdate = "'" + str(startdate) + "'" top['threads.last year'] = MLS.getLongestThreads( startdate, enddate, identities_db, npeople) startdate = datetime.date.today() - datetime.timedelta(days=30) startdate = "'" + str(startdate) + "'" top['threads.last month'] = MLS.getLongestThreads( startdate, enddate, identities_db, npeople) else: filter_name = filter_.get_name() item = filter_.get_item() if filter_name in [ "company", "domain", "repository", "domain", "country" ]: if filter_name in [ "company", "domain", "repository", "domain", "country" ]: top['senders.'] = msenders.get_list(mfilter, 0) top['senders.last month'] = msenders.get_list(mfilter, 31) top['senders.last year'] = msenders.get_list(mfilter, 365) else: # Remove filters above if there are performance issues top = msenders.get_list(mfilter) else: top = None return top
def __init__(self, dbcon = None, filters = None): """db connection and filter to be used""" self.db = dbcon self.filters = filters if filters == None: people_out = None organizations_out = None npeople = None type_analysis = None self.filters = MetricFilters(Metrics.default_period, Metrics.default_start, Metrics.default_end, type_analysis, npeople, people_out, organizations_out)
def GetCommitsSummaryCompanies(period, startdate, enddate, identities_db, num_organizations): # This function returns the following dataframe structrure # unixtime, date, week/month/..., company1, company2, ... company[num_organizations -1], others # The 3 first fields are used for data and ordering purposes # The "companyX" fields are those that provide info about that company # The "Others" field is the aggregated value of the rest of the organizations # Companies above num_organizations will be aggregated in Others from vizgrimoire.SCM import SCM metric = DataSource.get_metrics("organizations", SCM) organizations = metric.get_list() organizations = organizations['name'] first_organizations = {} count = 1 for company in organizations: company_name = "'" + company + "'" type_analysis = ['company', company_name] mcommits = DataSource.get_metrics("commits", SCM) mfilter = MetricFilters(period, startdate, enddate, type_analysis) mfilter_orig = mcommits.filters mcommits.filters = mfilter commits = mcommits.get_ts() mcommits.filters = mfilter_orig # commits = EvolCommits(period, startdate, enddate, identities_db, ["company", company_name]) # commits = completePeriodIds(commits, period, startdate, enddate) # Rename field commits to company name commits[company] = commits["commits"] del commits['commits'] if (count <= num_organizations): #Case of organizations with entity in the dataset first_organizations = dict(first_organizations.items() + commits.items()) else: #Case of organizations that are aggregated in the field Others if 'Others' not in first_organizations: first_organizations['Others'] = commits[company] else: first_organizations['Others'] = [ a + b for a, b in zip(first_organizations['Others'], commits[company]) ] count = count + 1 #TODO: remove global variables... first_organizations = completePeriodIds(first_organizations, period, startdate, enddate) return (first_organizations)
def get_trends(self, date, days): #Specific needs for Added and Removed lines not considered in meta class Metrics filters = self.filters chardates = GetDates(date, days) self.filters = MetricFilters(Metrics.default_period, chardates[1], chardates[0], None) last = self.get_agg() if last['added_lines'] is None: last['added_lines'] = 0 last_added = int(last['added_lines']) if last['removed_lines'] is None: last['removed_lines'] = 0 last_removed = int(last['removed_lines']) self.filters = MetricFilters(Metrics.default_period, chardates[2], chardates[1], None) prev = self.get_agg() if prev['added_lines'] is None: prev['added_lines'] = 0 prev_added = int(prev['added_lines']) if prev['removed_lines'] is None: prev['removed_lines'] = 0 prev_removed = int(prev['removed_lines']) data = {} data['diff_netadded_lines_' + str(days)] = last_added - prev_added data['percentage_added_lines_' + str(days)] = GetPercentageDiff( prev_added, last_added) data['diff_netremoved_lines_' + str(days)] = last_removed - prev_removed data['percentage_removed_lines_' + str(days)] = GetPercentageDiff( prev_removed, last_removed) data['added_lines_' + str(days)] = last_added data['removed_lines_' + str(days)] = last_removed #Returning filters to their original value self.filters = filters return (data)
def data_source_increment_activity(opts, people_out, affs_out): # Per data source, the increment or decrement of the activity is displayed dataset = {} data_sources = ["Gits", "Tickets", "Mailing Lists", "Gerrit", "Askbot", "IRC"] action = ["commits", "closed tickets", "sent emails", "submitted reviews", "posted questions", "messages"] net_values = [] rel_values = [] #percentage wrt the previous 365 days scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities) its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities) mls_dbcon = MLSQuery(opts.dbuser, opts.dbpassword, opts.dbmlstats, opts.dbidentities) scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities) qaforums_dbcon = QAForumsQuery(opts.dbuser, opts.dbpassword, opts.dbqaforums, opts.dbidentities) irc_dbcon = IRCQuery(opts.dbuser, opts.dbpassword, opts.dbirc, opts.dbidentities) period = "month" type_analysis = None releases = opts.releases.split(",")[-2:] startdate = "'"+releases[0]+"'" enddate = "'"+releases[1]+"'" filters = MetricFilters(period, startdate, enddate, None, 10, people_out, affs_out) commits = scm.Commits(scm_dbcon, filters) closed = its.Closed(its_dbcon, filters) emails = mls.EmailsSent(mls_dbcon, filters) submitted = scr.Submitted(scr_dbcon, filters) questions = qa.Questions(qaforums_dbcon, filters) messages = irc.Sent(irc_dbcon, filters) from vizgrimoire.ITS import ITS ITS.set_backend("jira") net_values.append(commits.get_trends(releases[1], 90)["commits_90"]) rel_values.append(commits.get_trends(releases[1], 90)["percentage_commits_90"]) net_values.append(closed.get_trends(releases[1], 90)["closed_90"]) rel_values.append(closed.get_trends(releases[1], 90)["percentage_closed_90"]) net_values.append(emails.get_trends(releases[1], 90)["sent_90"]) rel_values.append(emails.get_trends(releases[1], 90)["percentage_sent_90"]) net_values.append(submitted.get_trends(releases[1], 90)["submitted_90"]) rel_values.append(submitted.get_trends(releases[1], 90)["percentage_submitted_90"]) net_values.append(questions.get_trends(releases[1], 90)["qsent_90"]) rel_values.append(questions.get_trends(releases[1], 90)["percentage_qsent_90"]) net_values.append(messages.get_trends(releases[1], 90)["sent_90"]) rel_values.append(messages.get_trends(releases[1], 90)["percentage_sent_90"]) createCSV({"datasource":data_sources, "metricsnames":action, "relativevalues":rel_values, "netvalues":net_values}, "./release/data_source_evolution.csv")
def _get_data(period, startdate, enddate, i_db, filter_, evol): data = {} type_analysis = None if (filter_ is not None): type_analysis = [filter_.get_name(), filter_.get_item()] logging.warn(DownloadsDS.get_name() + " does not support filters.") return data if (evol): metrics_on = DownloadsDS.get_metrics_core_ts() else: metrics_on = DownloadsDS.get_metrics_core_agg() mfilter = MetricFilters(period, startdate, enddate, type_analysis) all_metrics = DownloadsDS.get_metrics_set(DownloadsDS) for item in all_metrics: if item.id not in metrics_on: continue item.filters = mfilter if evol is False: mvalue = item.get_agg() else: mvalue = item.get_ts() data = dict(data.items() + mvalue.items()) if evol is False: init_date = DownloadsDS.get_date_init(startdate, enddate, None, type_analysis) end_date = DownloadsDS.get_date_end(startdate, enddate, None, type_analysis) data = dict(data.items() + init_date.items() + end_date.items()) # Tendencies metrics_trends = DownloadsDS.get_metrics_core_trends() for i in [7, 30, 365]: for item in all_metrics: if item.id not in metrics_trends: continue period_data = item.get_trends(enddate, i) data = dict(data.items() + period_data.items()) if filter_ is not None: studies = {} else: studies = DataSource.get_studies_data(DownloadsDS, period, startdate, enddate, evol) return dict(data.items() + studies.items())
def get_list(self): # Just get commits per project startdate = self.filters.startdate enddate = self.filters.enddate type_analysis = ['project', None] period = None evol = False mcommits = Commits(self.db, self.filters) mfilter = MetricFilters(period, startdate, enddate, type_analysis) mfilter_orig = mcommits.filters mcommits.filters = mfilter commits = mcommits.get_agg() mcommits.filters = mfilter_orig checkListArray(commits) return commits
def get_top_data(startdate, enddate, identities_db, filter_, npeople): top = {} mauthors = DataSource.get_metrics("pulls", DockerHubDS) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is None: top['pulls.'] = mauthors.get_list(mfilter, 0) else: logging.info("DockerHubDS does not support yet top for filters.") return (top)
def get_list(self): # Just get closed per project startdate = self.filters.startdate enddate = self.filters.enddate type_analysis = ['project', None] period = None evol = False mclosed = Closed(self.db, self.filters) mfilter = MetricFilters(period, startdate, enddate, type_analysis) mfilter_orig = mclosed.filters mclosed.filters = mfilter closed = mclosed.get_agg() mclosed.filters = mfilter_orig checkListArray(closed) return closed
def get_list(self): # Just get sent per project startdate = self.filters.startdate enddate = self.filters.enddate type_analysis = ['project', None] period = None evol = False msent = EmailsSent(self.db, self.filters) mfilter = MetricFilters(period, startdate, enddate, type_analysis) mfilter_orig = msent.filters msent.filters = mfilter sent = msent.get_agg() msent.filters = mfilter_orig checkListArray(sent) return sent
def test_create_reports_studies(self): opts = read_options() period = getPeriod(opts.granularity) destdir = os.path.join("data","json") from vizgrimoire.metrics.metrics_filter import MetricFilters db_identities= Report.get_config()['generic']['db_identities'] dbuser = Report.get_config()['generic']['db_user'] dbpass = Report.get_config()['generic']['db_password'] studies = Report.get_studies() metric_filters = MetricFilters(period, startdate, enddate, []) for ds in Report.get_data_sources(): ds_dbname = ds.get_db_name() dbname = Report.get_config()['generic'][ds_dbname] dsquery = ds.get_query_builder() dbcon = dsquery(dbuser, dbpass, dbname, db_identities) for study in studies: # logging.info("Creating report for " + study.id + " for " + ds.get_name()) try: obj = study(dbcon, metric_filters) obj.create_report(ds, destdir) files = obj.get_report_files(ds) if len(files) > 0: for file in files: f_test_json = os.path.join("json", file) f_report_json = os.path.join(destdir, file) if obj.get_definition()['id'] == "contributors_new_gone": # authors is a dict with revtime which changes every execution pass else: self.assertTrue(self.compareJSON(f_test_json, f_report_json)) data = obj.result(ds) if data is None: continue test_json = os.path.join("json",ds.get_name()+"_"+obj.get_definition()['id']+".json") if obj.get_definition()['id'] == "top_issues": # Include time field which changes every execution continue else: self.assertTrue(self._compare_data(data, test_json)) except TypeError: traceback.print_exc(file=sys.stdout) logging.info(study.id + " does no support complete standard API. Not used when no available.") continue
def get_top_data (startdate, enddate, identities_db, filter_, npeople): top = {} mauthors = DataSource.get_metrics("authors", Mediawiki) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is None: top['authors.'] = mauthors.get_list(mfilter, 0) top['authors.last month'] = mauthors.get_list(mfilter, 31) top['authors.last year'] = mauthors.get_list(mfilter, 365) else: logging.info("Mediawiki does not support yet top for filters.") return(top)
def get_top_data(startdate, enddate, identities_db, filter_=None, npeople=None): def filter_ips(ips): new_ips = {} new_ips['downloads'] = ips['downloads'] new_ips['ips'] = [] for ip in ips['ips']: # ipv4 new_ip_aux = ip.split(".") new_ip = ip if len(new_ip_aux) == 4: new_ip = "x.x." + new_ip_aux[2] + "." + new_ip_aux[3] # ipv6 new_ip_aux = ip.split(":") if len(new_ip_aux) > 1: new_ip = new_ip_aux[0] + ":X" new_ips['ips'].append(new_ip) return new_ips top = {} mips = DataSource.get_metrics("ips", DownloadsDS) mpackages = DataSource.get_metrics("packages", DownloadsDS) mpages = DataSource.get_metrics("pages", DownloadsDS) mcountries = DataSource.get_metrics("countries", DownloadsDS) period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is None: top['ips.'] = filter_ips(mips.get_list(mfilter, 0)) top['packages.'] = mpackages.get_list(mfilter, 0) top['pages.'] = mpages.get_list() top['countries.'] = mcountries.get_list() else: logging.info("DownloadsDS does not support yet top for filters.") return top
def get_top_data_organizations(startdate, enddate, i_db, filter_, npeople): top = {} morganizations = DataSource.get_metrics("organizations", SCM) if morganizations is None: return top period = None if filter_ is not None: if filter_.get_name() == "project": # TODO missing organizations_out organizations_out = morganizations.filters.organizations_out people_out = None mfilter = MetricFilters(period, startdate, enddate, filter_.get_type_analysis(), npeople, people_out, organizations_out) mfilter_orig = morganizations.filters morganizations.filters = mfilter top = morganizations.get_list() morganizations.filters = mfilter_orig return top
def projects_efficiency(opts, people_out, affs_out): # BMI and time to review in mean per general project scr_dbcon = SCRQuery(opts.dbuser, opts.dbpassword, opts.dbreview, opts.dbidentities, opts.dbprojects) scm_dbcon = SCMQuery(opts.dbuser, opts.dbpassword, opts.dbcvsanaly, opts.dbidentities, opts.dbprojects) its_dbcon = ITSQuery(opts.dbuser, opts.dbpassword, opts.dbbicho, opts.dbidentities, opts.dbprojects) bmi_list = [] time2review_list = [] bmi_its = [] period = "month" releases = opts.releases.split(",")[-2:] startdate = "'"+releases[0]+"'" enddate = "'"+releases[1]+"'" type_analysis = None project_filters = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, affs_out) scr_bmi = scr.BMISCR(scr_dbcon, project_filters) time2review = scr.TimeToReview(scr_dbcon, project_filters) # ITS BMI index from vizgrimoire.ITS import ITS ITS.set_backend("jira") opened = its.Opened(its_dbcon, project_filters) closed = its.Closed(its_dbcon, project_filters) tickets_opened = opened.get_agg()["opened"] tickets_closed = closed.get_agg()["closed"] its_bmi = 0 if tickets_closed > 0: its_bmi = round(float(tickets_closed)/float(tickets_opened), 2) bmi_list.append(round(scr_bmi.get_agg()["bmiscr"], 2)) time2review_list.append(round(time2review.get_agg()["review_time_days_median"], 2)) bmi_its.append(its_bmi) createCSV({"projects":"general", "bmi":bmi_list, "timereview":time2review_list, "bmiits":bmi_its}, "./release/integrated_projects_efficiency.csv")
def get_top_data(startdate, enddate, identities_db, filter_, npeople): top = {} attendees = DataSource.get_metrics("rsvps", EventsDS) period = attendees.filters.period type_analysis = None mfilter = attendees.filters # updated filters filters = None # original filters if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if filter_ is not None: filters = attendees.filters attendees.filters = mfilter top['rsvps.'] = attendees.get_list(mfilter, 0) top['rsvps.last month'] = attendees.get_list(mfilter, 31) top['rsvps.last year'] = attendees.get_list(mfilter, 365) if filter_ is not None: attendees.filters = filters events = DataSource.get_metrics("events", EventsDS) if filter_ is not None: filters = events.filters events.filters = mfilter top['events.'] = events.get_list(mfilter, 0) top['events.last month'] = events.get_list(mfilter, 31) top['events.last year'] = events.get_list(mfilter, 365) if filter_ is not None: events.filters = filters if filter_ is not None: groups = DataSource.get_metrics("groups", EventsDS) filters = groups.filters groups.filters = mfilter if filter_.get_name() <> 'repository': top['groups.'] = groups.get_list(mfilter, 0) top['groups.last month'] = groups.get_list(mfilter, 31) top['groups.last year'] = groups.get_list(mfilter, 365) groups.filters = filters return top
def GetClosedSummaryCompanies(period, startdate, enddate, identities_db, closed_condition, num_organizations): from vizgrimoire.ITS import ITS count = 1 first_organizations = {} metric = DataSource.get_metrics("organizations", ITS) organizations = metric.get_list() organizations = organizations['name'] for company in organizations: type_analysis = ["company", "'" + company + "'"] filter_com = MetricFilters(period, startdate, enddate, type_analysis) mclosed = ITS.get_metrics("closed", ITS) mclosed.filters = filter_com closed = mclosed.get_ts() # Rename field closed to company name closed[company] = closed["closed"] del closed['closed'] if (count <= num_organizations): #Case of organizations with entity in the dataset first_organizations = dict(first_organizations.items() + closed.items()) else: #Case of organizations that are aggregated in the field Others if 'Others' not in first_organizations: first_organizations['Others'] = closed[company] else: first_organizations['Others'] = [ a + b for a, b in zip(first_organizations['Others'], closed[company]) ] count = count + 1 first_organizations = completePeriodIds(first_organizations, period, startdate, enddate) return (first_organizations)
def get_top_data (cls, startdate, enddate, identities_db, filter_, npeople): bots = cls.get_bots() closed_condition = cls._get_closed_condition() # TODO: It should be configurable from Automator top_issues_on = False top = None mopeners = DataSource.get_metrics("openers", cls) mclosers = DataSource.get_metrics("closers", cls) # We should check this metric is ON stories_openers = DataSource.get_metrics("stories_openers", cls) if mopeners is None or mclosers is None: return None period = None type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() mfilter = MetricFilters(period, startdate, enddate, type_analysis, npeople) if mclosers.filters.closed_condition is not None: mfilter.closed_condition = mclosers.filters.closed_condition if filter_ is None: top_closers_data = {} top_closers_data['closers.'] = mclosers.get_list(mfilter, 0) top_closers_data['closers.last month']= mclosers.get_list(mfilter, 31) top_closers_data['closers.last year']= mclosers.get_list(mfilter, 365) top_openers_data = {} top_openers_data['openers.'] = mopeners.get_list(mfilter, 0) top_openers_data['openers.last month'] = mopeners.get_list(mfilter, 31) top_openers_data['openers.last year'] = mopeners.get_list(mfilter, 365) top = dict(top_closers_data.items() + top_openers_data.items()) if top_issues_on: from vizgrimoire.analysis.top_issues import TopIssues from vizgrimoire.report import Report db_identities= Report.get_config()['generic']['db_identities'] dbuser = Report.get_config()['generic']['db_user'] dbpass = Report.get_config()['generic']['db_password'] dbname = Report.get_config()['generic'][cls.get_db_name()] dbcon = ITSQuery(dbuser, dbpass, dbname, db_identities) metric_filters = MetricFilters(None, startdate, enddate, []) top_issues_data = TopIssues(dbcon, metric_filters).result(cls) top = dict(top.items() + top_issues_data.items()) if False and stories_openers is not None: top_sopeners_data = {} top_sopeners_data['stories_openers.'] = stories_openers.get_list(mfilter, 0) top_sopeners_data['stories_openers.last month'] = stories_openers.get_list(mfilter, 31) top_sopeners_data['stories_openers.last year'] = stories_openers.get_list(mfilter, 365) top = dict(top.items() + top_sopeners_data.items()) else: filter_name = filter_.get_name() if filter_name in ["company","domain","repository"]: if filter_name in ["company","domain","repository"]: top = {} top['closers.'] = mclosers.get_list(mfilter, 0) top['closers.last month']= mclosers.get_list(mfilter, 31) top['closers.last year']= mclosers.get_list(mfilter, 365) else: # Remove filters above if there are performance issues top = mclosers.get_list(mfilter) else: top = None return top
def __get_data__ (period, startdate, enddate, identities_db, filter_ = None, evol = False): data = {} DS = SCR from vizgrimoire.GrimoireUtils import fill_and_order_items type_analysis = None if filter_ is not None: type_analysis = [filter_.get_name(), filter_.get_item()] from vizgrimoire.report import Report automator = Report.get_config() if evol: metrics_on = DS.get_metrics_core_ts() automator_metrics = DS.get_name()+"_metrics_ts" else: metrics_on = DS.get_metrics_core_agg() automator_metrics = DS.get_name()+"_metrics_agg" if automator_metrics in automator['r']: metrics_on = automator['r'][automator_metrics].split(",") logging.info(automator_metrics + " found ") # print(metrics_on) people_out = [] if "people_out" in Report.get_config()['r']: people_out = Report.get_config()['r']["people_out"] people_out = people_out.split(",") metrics_reports = SCR.get_metrics_core_reports() if filter_ is None: from vizgrimoire.report import Report reports_on = Report.get_config()['r']['reports'].split(",") for r in metrics_reports: if r in reports_on: metrics_on += [r] if type_analysis and type_analysis[1] is None: items = DS.get_filter_items(filter_, startdate, enddate, identities_db) items = items.pop('name') if DS.get_name()+"_start_date" in Report.get_config()['r']: startdate = "'"+Report.get_config()['r'][DS.get_name()+"_start_date"]+"'" if DS.get_name()+"_end_date" in Report.get_config()['r']: enddate = "'"+Report.get_config()['r'][DS.get_name()+"_end_date"]+"'" mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None) all_metrics = SCR.get_metrics_set(SCR) # SCR specific: remove some metrics from filters if filter_ is not None: metrics_not_filters = SCR.get_metrics_not_filters() metrics_on_filters = list(set(metrics_on) - set(metrics_not_filters)) if filter_.get_name() == "repository": if 'review_time' in metrics_on: metrics_on_filters+= ['review_time'] if 'submitted' in metrics_on: metrics_on_filters+= ['submitted'] metrics_on = metrics_on_filters # END SCR specific for item in all_metrics: if item.id not in metrics_on: continue # logging.info(item.id) mfilter_orig = item.filters mfilter.global_filter = mfilter_orig.global_filter item.filters = mfilter if not evol: mvalue = item.get_agg() else: mvalue = item.get_ts() if type_analysis and type_analysis[1] is None: logging.info(item.id) id_field = SCRQuery.get_group_field_alias(type_analysis[0]) mvalue = check_array_values(mvalue) mvalue = fill_and_order_items(items, mvalue, id_field, evol, period, startdate, enddate) data = dict(data.items() + mvalue.items()) item.filters = mfilter_orig # SCR SPECIFIC # if evol: if type_analysis and type_analysis[1] is None: pass else: metrics_on_changes = ['merged','abandoned','new'] for item in all_metrics: if item.id in metrics_on_changes and filter_ is None: mfilter_orig = item.filters item.filters = mfilter mvalue = item.get_ts_changes() data = dict(data.items() + mvalue.items()) item.filters = mfilter_orig # END SCR SPECIFIC # if not evol: init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis) end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis) if type_analysis and type_analysis[1] is None: if id_field is None: id_field = dsquery.get_group_field_alias(type_analysis[0]) init_date = fill_and_order_items(items, init_date, id_field, evol, period, startdate, enddate) end_date = fill_and_order_items(items, end_date, id_field, evol, period, startdate, enddate) data = dict(data.items() + init_date.items() + end_date.items()) # Tendencies metrics_trends = SCR.get_metrics_core_trends() automator_metrics = DS.get_name()+"_metrics_trends" if automator_metrics in automator['r']: metrics_trends = automator['r'][automator_metrics].split(",") for i in [7,30,365]: for item in all_metrics: if item.id not in metrics_trends: continue mfilter_orig = item.filters item.filters = mfilter period_data = item.get_trends(enddate, i) item.filters = mfilter_orig data = dict(data.items() + period_data.items()) if type_analysis and type_analysis[1] is None: id_field = SCRQuery.get_group_field_alias(type_analysis[0]) period_data = fill_and_order_items(items, period_data, id_field) data = dict(data.items() + period_data.items()) if filter_ is not None: studies_data = {} else: studies_data = DataSource.get_studies_data(SCR, period, startdate, enddate, evol) data = dict(data.items() + studies_data.items()) return data
def get_list (self): #TODO: warning -> not using GetSQLReportFrom/Where q = "SELECT re.url as name, COUNT(DISTINCT(pr.id)) AS submitted "+\ " FROM pull_requests pr, repositories re "+\ " WHERE pr.repo_id = re.id AND "+\ " pr.created_at >="+ self.filters.startdate+ " AND "+\ " pr.created_at < "+ self.filters.enddate +\ " GROUP BY re.id "+\ " ORDER BY submitted DESC, name " names = self.db.ExecuteQuery(q) if not isinstance(names['name'], (list)): names['name'] = [names['name']] return(names) if __name__ == '__main__': filters = MetricFilters("month", "'2014-01-01'", "'2015-01-01'") dbcon = PullpoQuery("root", "", "xxxxx", "xxxxx") timeto = TimeToClose(dbcon, filters) print timeto.get_agg() print timeto.get_ts() timeto = TimeToMerge(dbcon, filters) print timeto.get_agg() print timeto.get_ts() filters1 = filters.copy() type_analysis = ["company", "'xxxxx'"] filters1.type_analysis = type_analysis company = TimeToMerge(dbcon, filters1) print company.get_ts()
def get_metrics_data(DS, period, startdate, enddate, identities_db, filter_ = None, evol = False): """ Get basic data from all core metrics """ from vizgrimoire.GrimoireUtils import fill_and_order_items from vizgrimoire.ITS import ITS from vizgrimoire.MLS import MLS data = {} dsquery = DSQuery if DS == ITS: dsquery = ITSQuery if DS == MLS: dsquery = MLSQuery from vizgrimoire.report import Report automator = Report.get_config() if evol: metrics_on = DS.get_metrics_core_ts() automator_metrics = DS.get_name()+"_metrics_ts" else: metrics_on = DS.get_metrics_core_agg() automator_metrics = DS.get_name()+"_metrics_agg" if automator_metrics in automator['r']: metrics_on = automator['r'][automator_metrics].split(",") people_out = [] if "people_out" in Report.get_config()['r']: people_out = Report.get_config()['r']["people_out"] people_out = people_out.split(",") type_analysis = None if filter_ is not None: type_analysis = filter_.get_type_analysis() if type_analysis and type_analysis[1] is None: # We need the items for filling later values in group by queries items = DS.get_filter_items(filter_, startdate, enddate, identities_db) if items is None: return data items = items.pop('name') if DS.get_name()+"_startdate" in Report.get_config()['r']: startdate = Report.get_config()['r'][DS.get_name()+"_startdate"] if DS.get_name()+"_enddate" in Report.get_config()['r']: enddate = Report.get_config()['r'][DS.get_name()+"_enddate"] # TODO: the hardcoded 10 should be removed, and use instead the npeople provided # in the config file. mfilter = MetricFilters(period, startdate, enddate, type_analysis, 10, people_out, None) metrics_reports = DS.get_metrics_core_reports() all_metrics = DS.get_metrics_set(DS) # Reports = filters metrics not available inside filters if type_analysis is None: from vizgrimoire.report import Report reports_on = Report.get_config()['r']['reports'].split(",") for r in metrics_reports: if r in reports_on: metrics_on += [r] for item in all_metrics: # print item if item.id not in metrics_on: continue mfilter_orig = item.filters mfilter.global_filter = mfilter_orig.global_filter mfilter.set_closed_condition(mfilter_orig.closed_condition) item.filters = mfilter if evol: mvalue = item.get_ts() else: mvalue = item.get_agg() if type_analysis and type_analysis[1] is None and mvalue: logging.info(item.id) id_field = None # Support for combined filters for idf in mvalue.keys(): if "CONCAT(" in idf: id_field = idf break if id_field is None: id_field = dsquery.get_group_field_alias(type_analysis[0]) mvalue = fill_and_order_items(items, mvalue, id_field, evol, period, startdate, enddate) data = dict(data.items() + mvalue.items()) item.filters = mfilter_orig if not evol: init_date = DS.get_date_init(startdate, enddate, identities_db, type_analysis) end_date = DS.get_date_end(startdate, enddate, identities_db, type_analysis) if type_analysis and type_analysis[1] is None: if id_field is None: id_field = dsquery.get_group_field_alias(type_analysis[0]) init_date = fill_and_order_items(items, init_date, id_field, evol, period, startdate, enddate) end_date = fill_and_order_items(items, end_date, id_field, evol, period, startdate, enddate) if init_date is None: init_date = {} if end_date is None: end_date = {} data = dict(data.items() + init_date.items() + end_date.items()) # Tendencies metrics_trends = DS.get_metrics_core_trends() automator_metrics = DS.get_name()+"_metrics_trends" if automator_metrics in automator['r']: metrics_trends = automator['r'][automator_metrics].split(",") for i in [7,30,365]: for item in all_metrics: if item.id not in metrics_trends: continue mfilter_orig = item.filters item.filters = mfilter period_data = item.get_trends(enddate, i) item.filters = mfilter_orig if type_analysis and type_analysis[1] is None: group_field = dsquery.get_group_field_alias(type_analysis[0]) period_data = fill_and_order_items(items, period_data, group_field) data = dict(data.items() + period_data.items()) return data