def social_chart(self):
current = self.period.start_date
end = self.period.end_date
dates = []
while current <= end:
current_range = utils.StatsRange("day", current, current)
dates.append(current_range)
current = current + timedelta(days=1)
data = {}
network_data = {}
for count, date in enumerate(dates):
social = []
network_social = []
metrics = "ga:pageviews,ga:users,ga:sessions"
for site in config.TABLES.keys():
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters="ga:socialNetwork!=(not set)",
sort="-ga:users")]
if rows[0]:
rows = self._remove_ga_names(rows)
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
if site in self.sites:
social.extend(rows)
network_social.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregate = utils.aggregate_data(social, ["pageviews", "users", "sessions"])
network_aggregate = utils.aggregate_data(network_social, ["pageviews", "users", "sessions"])
data[date.get_start()] = aggregate
network_data[date.get_start()] = network_aggregate
x_labels = []
graph_data = {"users":[], "pageviews":[], "network_pageviews":[], "network_users":[]}
for range in dates:
x_labels.append(range.get_start())
graph_data["users"].append(data[range.get_start()]["users"])
graph_data["pageviews"].append(data[range.get_start()]["pageviews"])
graph_data["network_users"].append(network_data[range.get_start()]["users"])
graph_data["network_pageviews"].append(network_data[range.get_start()]["pageviews"])
chart = utils.chart("Social Data", x_labels, graph_data, "Day", "Number")
return chart
def rollup_ids(self,
ids,
start,
end,
metrics,
dimensions=None,
filters=None,
sort=None,
max_results=None,
aggregate_key=None):
main_row = []
for id in ids:
if filters:
filter = filters + ";channel==%s" % id
else:
filter = "channel==%s" % id
results = self.run_analytics_report(start,
end,
metrics=metrics,
dimensions=dimensions,
filters=filter,
max_results=max_results,
sort=sort)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start +
" - " + end)
main_row = utils.aggregate_data(main_row, metrics.split(","),
aggregate_key)
return main_row
def summary_table(self):
data = {}
for count, date in enumerate(self.date_list):
totals = []
metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
for site in self.sites:
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]
if rows[0]:
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
totals.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregate = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"])
data[count] = aggregate
for period in data:
data[period]["pv_per_session"] = data[period].get("pv_per_session", 0)/len(self.sites)
data[period]["avg_session_time"] = (data[period].get("avg_session_time", 0)/len(self.sites))/60.0
this_period = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous")
this_period = utils.add_change(this_period, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly")
return this_period
def social_network_table(self, num_articles):
data = {}
for count, date in enumerate(self.date_list):
social = []
metrics = "ga:pageviews,ga:users,ga:sessions"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:socialNetwork", filters="ga:socialNetwork!=(not set)",
sort="-ga:users", aggregate_key="ga:socialNetwork")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"social_network":"socialNetwork"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
social.extend(rows)
aggregated = utils.aggregate_data(social, ["pageviews", "users", "sessions"], match_key="social_network")
sorted = utils.sort_data(aggregated, "users", limit=15)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions"], "previous", match_key="social_network")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions"], "yearly", match_key="social_network")
for row in added_change:
filter = "ga:socialNetwork==%s" % row["social_network"]
article = self.referral_articles(filter, num_articles)
row["articles"] = article
return added_change
def referral_articles(self, filter, limit):
filters = config.ARTICLE_FILTER + ";" + filter
article_previous = utils.StatsRange.get_previous_period(self.period, "DAILY")#how to do this
data = {}
for count, date in enumerate([self.period, article_previous]):
articles = []
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:pageviews", dimensions="ga:pageTitle,ga:pagePath,ga:hostname", filters=filters,
sort="-ga:pageviews", aggregate_key="ga:pagePath")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"title":"pageTitle", "path":"pagePath", "host":"hostname"})
for row in rows:
path = row["path"]
title = row["title"]
new_path = self._remove_query_string(path)
new_title = self._get_title(path, title)
row["path"] = new_path
row["title"] = new_title
row["pageviews"] = float(row["pageviews"])
articles.extend(rows)
aggregated = utils.aggregate_data(articles, ["pageviews"], match_key="path")
sorted = utils.sort_data(aggregated, "pageviews", limit=limit)
data[count] = sorted
#group
added_change = utils.add_change(data[0], data[1], ["pageviews"], "previous", match_key="path")
return added_change
def site_summary_table(self):
#should this be combined with overall summary table?
data = {}
for count, date in enumerate(self.date_list):
totals = []
metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
for site in self.sites:
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["ga:site"] = site
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
totals.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], match_key="site")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous", match_key="site")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly", match_key="site")
return added_change
def _get_by_source(self, subdivide_by_medium=False):
data = {}
aggregate_key = "ga:source"
match_key = "source"
if subdivide_by_medium:
aggregate_key = "ga:sourceMedium"
match_key = "source_medium"
for count, date in enumerate(self.date_list):
traffic_sources = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=aggregate_key, sort="-ga:users", aggregate_key=aggregate_key)
rows = self._remove_ga_names(rows)
if subdivide_by_medium:
rows = utils.change_key_names(rows, {"source_medium":"sourceMedium"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users"])
traffic_sources.extend(rows)
aggregated = utils.aggregate_data(traffic_sources, ["pageviews", "users"], match_key=match_key)
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key=match_key)
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key=match_key)
return added_change
def video_table(self):
data = {}
for count, date in enumerate([self.period]):
table = []
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched,views", dimensions="video", filters=None, max_results="20",
sort="-estimatedMinutesWatched", aggregate_key="video")
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched", "views"])
video_name = analytics.get_video(row['video'])
try:
row['title'] = video_name['items'][0]['snippet']['title']
except IndexError:
row["title"] = "Video Not Found"
row['channel'] = channel
table.extend(rows)
aggregated = utils.aggregate_data(table, ["estimatedMinutesWatched", "views"], match_key="video")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=10)
data[count] = sorted
#group
return data[0]
def rollup_ids(self,
properties,
start,
end,
metrics,
dimensions=None,
filters=None,
sort=None,
max_results=None,
aggregate_key=None):
main_row = []
for property_details in properties:
id = property_details['id']
results = self.run_report(id,
start,
end,
metrics=metrics,
dimensions=dimensions,
filters=filters,
sort=sort,
max_results=max_results)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#no data
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start +
" - " + end)
str_dimensions = "None"
if dimensions:
str_dimensions = dimensions.encode('utf-8')
str_filters = "None"
if filters:
str_filters = filters.encode('utf-8')
logger.debug("for metrics: " + metrics + " dimensions: " +
str_dimensions + " filters: " + str_filters)
main_row = utils.aggregate_data(main_row, metrics.split(","),
aggregate_key)
return main_row
def device_table(self):
data = {}
for count, date in enumerate(self.date_list):
devices = []
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:users", dimensions="ga:deviceCategory", sort="-ga:users", aggregate_key="ga:deviceCategory")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"device_category":"deviceCategory"})
for row in rows:
row["users"] = float(row["users"])
devices.extend(rows)
aggregated = utils.aggregate_data(devices, ["users"], match_key="device_category")
sorted = utils.sort_data(aggregated, "users", limit=6)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["users"], "previous", match_key="device_category")
added_change = utils.add_change(added_change, data[2], ["users"], "yearly", match_key="device_category")
return added_change
def country_table(self):
countries = ["Czec", "Germa", "Denma", "Spai", "Franc", "Italy",
"Portug", "Swede", "Polan", "Brazi", "Belgiu", "Netherl",
"United Ki", "Irela", "United St", "Canad", "Austral", "New Ze"]
countries_regex = "|".join(countries)
filters = "ga:country=~%s" % countries_regex
row_filters = "ga:country!~%s" % countries_regex
data = {}
for count, date in enumerate(self.date_list):
breakdown = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:country", filters=filters,
sort="-ga:pageviews", aggregate_key="ga:country")
world_rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters=row_filters,
sort="-ga:pageviews", aggregate_key=None)]
if world_rows[0]:
world_rows[0]["ga:country"] = "ROW"
else:
world_rows = [{"ga:country":"ROW", "ga:pageviews":0, "ga:users":0}]
rows.extend(world_rows)
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
rows = self._remove_ga_names(rows)
breakdown.extend(rows)
aggregated = utils.aggregate_data(breakdown, ["pageviews", "users"], match_key="country")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key="country")
return added_change
def channel_summary_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics="subscribersGained,subscribersLost,estimatedMinutesWatched"
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
if count == 0:
subscriber_count = analytics.rollup_stats(ids)['subscriberCount'] #only gets the current subscriber count
elif count == 1:
#last period date, work out last periods subscriber count from current periods sub change
this_channel = utils.list_search(data[0], "channel", channel)
subscriber_count = this_channel["subscriberCount"] - this_channel["subscriberChange"]
else:
#don't need to work out yearly sub change
subscriber_count = 0.0
rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["channel"] = channel
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
row["subscriberCount"] = float(subscriber_count)
table.extend(rows)
else:
print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
#logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(table, ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], match_key= "channel")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], "previous", match_key= "channel")
added_change = utils.add_change(added_change, data[2], ["subscriberChange", "estimatedMinutesWatched"], "yearly", match_key= "channel")
return added_change
def country_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics = "views,estimatedMinutesWatched,subscribersGained,subscribersLost"
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions="country",
filters=None, max_results=None, sort="-estimatedMinutesWatched", aggregate_key="country")
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
table.extend(rows)
aggregated = utils.aggregate_data(table, (metrics + ",subscriberChange").split(","), match_key="country")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=20)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["views","estimatedMinutesWatched","subscriberChange"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["views","estimatedMinutesWatched","subscriberChange"], "yearly", match_key="country")
return added_change
def rollup_ids(self, properties, start, end, metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None):
main_row = []
for property_details in properties:
id = property_details['id']
results = self.run_report(id, start, end, metrics=metrics, dimensions=dimensions, filters=filters, sort=sort, max_results=max_results)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#no data
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start + " - " + end)
str_dimensions = "None"
if dimensions:
str_dimensions = dimensions.encode('utf-8')
str_filters = "None"
if filters:
str_filters = filters.encode('utf-8')
logger.debug("for metrics: " + metrics + " dimensions: " + str_dimensions + " filters: " + str_filters)
main_row = utils.aggregate_data(main_row, metrics.split(","), aggregate_key)
return main_row