def aggData(period, startdate, enddate, identities_db, destdir):
# data = vizr.StaticMLSInfo(period, startdate, enddate, identities_db, rfield)
# agg = dataFrame2Dict(data)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db, rfield)
agg = data
if ('companies' in reports):
data = MLS.AggMLSCompanies(period, startdate, enddate, identities_db)
agg = dict(agg.items() + data.items())
if ('countries' in reports):
data = MLS.AggMLSCountries(period, startdate, enddate, identities_db)
agg = dict(agg.items() + data.items())
if ('domains' in reports):
data = MLS.AggMLSDomains(period, startdate, enddate, identities_db)
agg = dict(agg.items() + data.items())
# Tendencies
for i in [7, 30, 365]:
# period_data = dataFrame2Dict(vizr.GetDiffSentDays(period, enddate, i))
period_data = MLS.GetDiffSentDays(period, enddate, i)
agg = dict(agg.items() + period_data.items())
period_data = MLS.GetDiffSendersDays(period, enddate, i)
agg = dict(agg.items() + period_data.items())
# Last Activity: to be removed
for i in [7, 14, 30, 60, 90, 180, 365, 730]:
period_activity = MLS.lastActivity(i)
agg = dict(agg.items() + period_activity.items())
createJSON(agg, destdir + "/mls-static.json")
def domainsData(period, startdate, enddate, identities_db, destdir, npeople):
domains = MLS.domainsNames(identities_db, startdate, enddate)
createJSON(domains, destdir + "/mls-domains.json")
for domain in domains:
domain_name = "'" + domain + "'"
type_analysis = ["domain", domain_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
data = completePeriodIds(data)
if (domain == "everybody" or domain == "hallowelt"):
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + domain + "-mls-dom-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + domain + "-mls-dom-evolutionary.json")
data = MLS.domainTopSenders(domain, identities_db, startdate, enddate,
npeople)
createJSON(data, destdir + "/" + domain + "-mls-dom-top-senders.json",
False)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
createJSON(data, destdir + "/" + domain + "-mls-dom-static.json")
def companiesData(period, startdate, enddate, identities_db, destdir, npeople):
# companies = valRtoPython(vizr.companiesNames(identities_db, startdate, enddate))
companies = MLS.companiesNames(identities_db, startdate, enddate)
createJSON(companies, destdir + "/mls-companies.json")
for company in companies:
company_name = "'" + company + "'"
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, ["company", company_name])
data = completePeriodIds(data)
if company in ["company4", "Deutsche Telekom", "IBM"]:
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + company + "-mls-com-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + company + "-mls-com-evolutionary.json")
top_senders = MLS.companyTopSenders(company, identities_db, startdate,
enddate, npeople)
createJSON(top_senders,
destdir + "/" + company + "-mls-com-top-senders.json",
False)
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, ["company", company_name])
createJSON(data, destdir + "/" + company + "-mls-com-static.json")
sent = MLS.GetSentSummaryCompanies(period, startdate, enddate,
opts.identities_db, 10)
createJSON(sent, opts.destdir + "/mls-sent-companies-summary.json")
def countriesData(period, startdate, enddate, identities_db, destdir, npeople):
countries = MLS.countriesNames(identities_db, startdate, enddate)
createJSON(countries, destdir + "/mls-countries.json")
for country in countries:
country_name = "'" + country + "'"
type_analysis = ["country", country_name]
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
data = completePeriodIds(data)
if (country == "country5" or country == "country2"):
# Wrong JSON generated in R. Don't check
createJSON(data,
destdir + "/" + country + "-mls-cou-evolutionary.json",
False)
else:
createJSON(data,
destdir + "/" + country + "-mls-cou-evolutionary.json")
top_senders = MLS.countryTopSenders(country, identities_db, startdate,
enddate, npeople)
createJSON(top_senders,
destdir + "/" + country + "-mls-cou-top-senders.json")
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, type_analysis)
createJSON(data, destdir + "/" + country + "-mls-cou-static.json")
def reposData(period, startdate, enddate, identities_db, destdir, conf,
repofield, npeople):
repos = MLS.reposNames(rfield, startdate, enddate)
createJSON(repos, destdir + "/mls-lists.json")
repos = repos['mailing_list_url']
check = True
if not isinstance(repos, (list)):
repos = [repos]
check = False
repos_files = [
repo.replace('/', '_').replace("<", "__").replace(">", "___")
for repo in repos
]
createJSON(repos_files, destdir + "/mls-repos.json", check)
for repo in repos:
# Evol data
repo_name = "'" + repo + "'"
data = MLS.EvolMLSInfo(period, startdate, enddate, identities_db,
rfield, ["repository", repo_name])
data = completePeriodIds(data)
listname_file = repo.replace("/",
"_").replace("<",
"__").replace(">", "___")
# TODO: Multilist approach. We will obsolete it in future
createJSON(
data, destdir + "/mls-" + listname_file + "-rep-evolutionary.json")
# Multirepos filename
createJSON(
data, destdir + "/" + listname_file + "-mls-rep-evolutionary.json")
top_senders = MLS.repoTopSenders(repo, identities_db, startdate,
enddate, repofield, npeople)
createJSON(top_senders,
destdir + "/" + listname_file + "-mls-rep-top-senders.json",
False)
# Static data
data = MLS.StaticMLSInfo(period, startdate, enddate, identities_db,
rfield, ["repository", repo_name])
# TODO: Multilist approach. We will obsolete it in future
createJSON(data, destdir + "/" + listname_file + "-rep-static.json")
# Multirepos filename
createJSON(data,
destdir + "/" + listname_file + "-mls-rep-static.json")
