def aggData(period, startdate, enddate, identities_db, destdir): # data = vizr.StaticMLSInfo(period, startdate, enddate, identities_db, rfield) # agg = dataFrame2Dict(data) data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield) agg = data if ('companies' in reports): data = MLS.AggMLSCompanies(period, startdate, enddate, identities_db) agg = dict(agg.items() + data.items()) if ('countries' in reports): data = MLS.AggMLSCountries(period, startdate, enddate, identities_db) agg = dict(agg.items() + data.items()) if ('domains' in reports): data = MLS.AggMLSDomains(period, startdate, enddate, identities_db) agg = dict(agg.items() + data.items()) # Tendencies for i in [7, 30, 365]: # period_data = dataFrame2Dict(vizr.GetDiffSentDays(period, enddate, i)) period_data = MLS.GetDiffSentDays(period, enddate, i) agg = dict(agg.items() + period_data.items()) period_data = MLS.GetDiffSendersDays(period, enddate, i) agg = dict(agg.items() + period_data.items()) # Last Activity: to be removed for i in [7, 14, 30, 60, 90, 180, 365, 730]: period_activity = MLS.lastActivity(i) agg = dict(agg.items() + period_activity.items()) createJSON(agg, destdir + "/mls-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir, npeople): domains = MLS.domainsNames(identities_db, startdate, enddate) createJSON(domains, destdir + "/mls-domains.json") for domain in domains: domain_name = "'" + domain + "'" type_analysis = ["domain", domain_name] data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis) data = completePeriodIds(data) if (domain == "everybody" or domain == "hallowelt"): # Wrong JSON generated in R. Don't check createJSON(data, destdir + "/" + domain + "-mls-dom-evolutionary.json", False) else: createJSON(data, destdir + "/" + domain + "-mls-dom-evolutionary.json") data = MLS.domainTopSenders(domain, identities_db, startdate, enddate, npeople) createJSON(data, destdir + "/" + domain + "-mls-dom-top-senders.json", False) data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis) createJSON(data, destdir + "/" + domain + "-mls-dom-static.json")
def companiesData(period, startdate, enddate, identities_db, destdir, npeople): # companies = valRtoPython(vizr.companiesNames(identities_db, startdate, enddate)) companies = MLS.companiesNames(identities_db, startdate, enddate) createJSON(companies, destdir + "/mls-companies.json") for company in companies: company_name = "'" + company + "'" data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, ["company", company_name]) data = completePeriodIds(data) if company in ["company4", "Deutsche Telekom", "IBM"]: # Wrong JSON generated in R. Don't check createJSON(data, destdir + "/" + company + "-mls-com-evolutionary.json", False) else: createJSON(data, destdir + "/" + company + "-mls-com-evolutionary.json") top_senders = MLS.companyTopSenders(company, identities_db, startdate, enddate, npeople) createJSON(top_senders, destdir + "/" + company + "-mls-com-top-senders.json", False) data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, ["company", company_name]) createJSON(data, destdir + "/" + company + "-mls-com-static.json") sent = MLS.GetSentSummaryCompanies(period, startdate, enddate, opts.identities_db, 10) createJSON(sent, opts.destdir + "/mls-sent-companies-summary.json")
def countriesData(period, startdate, enddate, identities_db, destdir, npeople): countries = MLS.countriesNames(identities_db, startdate, enddate) createJSON(countries, destdir + "/mls-countries.json") for country in countries: country_name = "'" + country + "'" type_analysis = ["country", country_name] data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis) data = completePeriodIds(data) if (country == "country5" or country == "country2"): # Wrong JSON generated in R. Don't check createJSON(data, destdir + "/" + country + "-mls-cou-evolutionary.json", False) else: createJSON(data, destdir + "/" + country + "-mls-cou-evolutionary.json") top_senders = MLS.countryTopSenders(country, identities_db, startdate, enddate, npeople) createJSON(top_senders, destdir + "/" + country + "-mls-cou-top-senders.json") data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, type_analysis) createJSON(data, destdir + "/" + country + "-mls-cou-static.json")
def reposData(period, startdate, enddate, identities_db, destdir, conf, repofield, npeople): repos = MLS.reposNames(rfield, startdate, enddate) createJSON(repos, destdir + "/mls-lists.json") repos = repos['mailing_list_url'] check = True if not isinstance(repos, (list)): repos = [repos] check = False repos_files = [ repo.replace('/', '_').replace("<", "__").replace(">", "___") for repo in repos ] createJSON(repos_files, destdir + "/mls-repos.json", check) for repo in repos: # Evol data repo_name = "'" + repo + "'" data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db, rfield, ["repository", repo_name]) data = completePeriodIds(data) listname_file = repo.replace("/", "_").replace("<", "__").replace(">", "___") # TODO: Multilist approach. We will obsolete it in future createJSON( data, destdir + "/mls-" + listname_file + "-rep-evolutionary.json") # Multirepos filename createJSON( data, destdir + "/" + listname_file + "-mls-rep-evolutionary.json") top_senders = MLS.repoTopSenders(repo, identities_db, startdate, enddate, repofield, npeople) createJSON(top_senders, destdir + "/" + listname_file + "-mls-rep-top-senders.json", False) # Static data data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield, ["repository", repo_name]) # TODO: Multilist approach. We will obsolete it in future createJSON(data, destdir + "/" + listname_file + "-rep-static.json") # Multirepos filename createJSON(data, destdir + "/" + listname_file + "-mls-rep-static.json")