def social_network_table(self, num_articles):
data = {}
for count, date in enumerate(self.date_list):
social = []
metrics = "ga:pageviews,ga:users,ga:sessions"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:socialNetwork", filters="ga:socialNetwork!=(not set)",
sort="-ga:users", aggregate_key="ga:socialNetwork")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"social_network":"socialNetwork"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
social.extend(rows)
aggregated = utils.aggregate_data(social, ["pageviews", "users", "sessions"], match_key="social_network")
sorted = utils.sort_data(aggregated, "users", limit=15)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions"], "previous", match_key="social_network")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions"], "yearly", match_key="social_network")
for row in added_change:
filter = "ga:socialNetwork==%s" % row["social_network"]
article = self.referral_articles(filter, num_articles)
row["articles"] = article
return added_change
def referral_articles(self, filter, limit):
filters = config.ARTICLE_FILTER + ";" + filter
article_previous = utils.StatsRange.get_previous_period(self.period, "DAILY")#how to do this
data = {}
for count, date in enumerate([self.period, article_previous]):
articles = []
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:pageviews", dimensions="ga:pageTitle,ga:pagePath,ga:hostname", filters=filters,
sort="-ga:pageviews", aggregate_key="ga:pagePath")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"title":"pageTitle", "path":"pagePath", "host":"hostname"})
for row in rows:
path = row["path"]
title = row["title"]
new_path = self._remove_query_string(path)
new_title = self._get_title(path, title)
row["path"] = new_path
row["title"] = new_title
row["pageviews"] = float(row["pageviews"])
articles.extend(rows)
aggregated = utils.aggregate_data(articles, ["pageviews"], match_key="path")
sorted = utils.sort_data(aggregated, "pageviews", limit=limit)
data[count] = sorted
#group
added_change = utils.add_change(data[0], data[1], ["pageviews"], "previous", match_key="path")
return added_change
def video_table(self):
data = {}
for count, date in enumerate([self.period]):
table = []
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched,views", dimensions="video", filters=None, max_results="20",
sort="-estimatedMinutesWatched", aggregate_key="video")
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched", "views"])
video_name = analytics.get_video(row['video'])
try:
row['title'] = video_name['items'][0]['snippet']['title']
except IndexError:
row["title"] = "Video Not Found"
row['channel'] = channel
table.extend(rows)
aggregated = utils.aggregate_data(table, ["estimatedMinutesWatched", "views"], match_key="video")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=10)
data[count] = sorted
#group
return data[0]
def _get_by_source(self, subdivide_by_medium=False):
data = {}
aggregate_key = "ga:source"
match_key = "source"
if subdivide_by_medium:
aggregate_key = "ga:sourceMedium"
match_key = "source_medium"
for count, date in enumerate(self.date_list):
traffic_sources = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=aggregate_key, sort="-ga:users", aggregate_key=aggregate_key)
rows = self._remove_ga_names(rows)
if subdivide_by_medium:
rows = utils.change_key_names(rows, {"source_medium":"sourceMedium"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users"])
traffic_sources.extend(rows)
aggregated = utils.aggregate_data(traffic_sources, ["pageviews", "users"], match_key=match_key)
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key=match_key)
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key=match_key)
return added_change
def site_summary_table(self):
#should this be combined with overall summary table?
data = {}
for count, date in enumerate(self.date_list):
totals = []
metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
for site in self.sites:
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["ga:site"] = site
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
totals.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], match_key="site")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous", match_key="site")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly", match_key="site")
return added_change
def traffic_source_table(self):
data = {}
source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
traffic_source = []
for source in source_types:
row = {"insightTrafficSourceType":source, "source_total":0.0}
traffic_source.append(row)
for count, date in enumerate(self.date_list):
table = []
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None,
sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
channel_total = 0.0
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
row["channel"] = channel
channel_total += row["estimatedMinutesWatched"]
new_rows = []
for source_type in traffic_source:
try:
result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
result["channel_total"] = channel_total
new_rows.append(result)
source_type["source_total"] += result["estimatedMinutesWatched"]
except KeyError:
new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})
table.append(new_rows)
new_table = []
for channel in table:
for row in channel:
#order each channel section by the total minutes watched for each source
#calculate channels percentage breakdown by source
breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
row["source_percentage"] = breakdown
result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
row["source_total"] = result["source_total"]
sort_row = utils.sort_data(channel, "source_total")
new_table.append(sort_row)
#sort the new table of dictionaries by total channel watch time
sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
data[count] = sorted_list
return data[0]
def traffic_source_table(self):
data = {}
source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
traffic_source = []
for source in source_types:
row = {"insightTrafficSourceType":source, "source_total":0.0}
traffic_source.append(row)
for count, date in enumerate(self.date_list):
table = []
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None,
sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
channel_total = 0.0
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
row["channel"] = channel
channel_total += row["estimatedMinutesWatched"]
new_rows = []
for source_type in traffic_source:
try:
result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
result["channel_total"] = channel_total
new_rows.append(result)
source_type["source_total"] += result["estimatedMinutesWatched"]
except KeyError:
new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})
table.append(new_rows)
new_table = []
for channel in table:
for row in channel:
#order each channel section by the total minutes watched for each source
#calculate channels percentage breakdown by source
breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
row["source_percentage"] = breakdown
result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
row["source_total"] = result["source_total"]
sort_row = utils.sort_data(channel, "source_total")
new_table.append(sort_row)
#sort the new table of dictionaries by total channel watch time
sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
data[count] = sorted_list
return data[0]
def device_table(self):
data = {}
for count, date in enumerate(self.date_list):
devices = []
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:users", dimensions="ga:deviceCategory", sort="-ga:users", aggregate_key="ga:deviceCategory")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"device_category":"deviceCategory"})
for row in rows:
row["users"] = float(row["users"])
devices.extend(rows)
aggregated = utils.aggregate_data(devices, ["users"], match_key="device_category")
sorted = utils.sort_data(aggregated, "users", limit=6)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["users"], "previous", match_key="device_category")
added_change = utils.add_change(added_change, data[2], ["users"], "yearly", match_key="device_category")
return added_change
def country_table(self):
countries = ["Czec", "Germa", "Denma", "Spai", "Franc", "Italy",
"Portug", "Swede", "Polan", "Brazi", "Belgiu", "Netherl",
"United Ki", "Irela", "United St", "Canad", "Austral", "New Ze"]
countries_regex = "|".join(countries)
filters = "ga:country=~%s" % countries_regex
row_filters = "ga:country!~%s" % countries_regex
data = {}
for count, date in enumerate(self.date_list):
breakdown = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:country", filters=filters,
sort="-ga:pageviews", aggregate_key="ga:country")
world_rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters=row_filters,
sort="-ga:pageviews", aggregate_key=None)]
if world_rows[0]:
world_rows[0]["ga:country"] = "ROW"
else:
world_rows = [{"ga:country":"ROW", "ga:pageviews":0, "ga:users":0}]
rows.extend(world_rows)
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
rows = self._remove_ga_names(rows)
breakdown.extend(rows)
aggregated = utils.aggregate_data(breakdown, ["pageviews", "users"], match_key="country")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key="country")
return added_change
def channel_summary_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics="subscribersGained,subscribersLost,estimatedMinutesWatched"
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
if count == 0:
subscriber_count = analytics.rollup_stats(ids)['subscriberCount'] #only gets the current subscriber count
elif count == 1:
#last period date, work out last periods subscriber count from current periods sub change
this_channel = utils.list_search(data[0], "channel", channel)
subscriber_count = this_channel["subscriberCount"] - this_channel["subscriberChange"]
else:
#don't need to work out yearly sub change
subscriber_count = 0.0
rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["channel"] = channel
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
row["subscriberCount"] = float(subscriber_count)
table.extend(rows)
else:
print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
#logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(table, ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], match_key= "channel")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], "previous", match_key= "channel")
added_change = utils.add_change(added_change, data[2], ["subscriberChange", "estimatedMinutesWatched"], "yearly", match_key= "channel")
return added_change
def country_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics = "views,estimatedMinutesWatched,subscribersGained,subscribersLost"
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions="country",
filters=None, max_results=None, sort="-estimatedMinutesWatched", aggregate_key="country")
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
table.extend(rows)
aggregated = utils.aggregate_data(table, (metrics + ",subscriberChange").split(","), match_key="country")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=20)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["views","estimatedMinutesWatched","subscriberChange"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["views","estimatedMinutesWatched","subscriberChange"], "yearly", match_key="country")
return added_change
def channel_stats_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
#just subscribersGained to get number of subscribers per 1000 views
metrics="views,likes,dislikes,comments,shares,subscribersGained"
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions=None)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["channel"] = channel
row["likeRate"] = utils.rate_per_1000(row["likes"], row['views'])
row["commentRate"] = utils.rate_per_1000(row["comments"], row['views'])
row["sharesRate"] = utils.rate_per_1000(row["shares"], row['views'])
row["subsRate"] = utils.rate_per_1000(row["subscribersGained"], row['views'])
try:
row["likeRatio"] = utils.sig_fig(2, row["likes"] / row["dislikes"])
except ZeroDivisionError:
row["likeRatio"] = 0
try:
row["dislikeRatio"] = utils.sig_fig(2, row["dislikes"] / row["dislikes"])
except ZeroDivisionError:
row["dislikeRatio"] = 0
table.extend(rows)
else:
#print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end())
sorted = utils.sort_data(table, "views")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "previous", match_key= "channel")
added_change = utils.add_change(added_change, data[2], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "yearly", match_key= "channel")
return added_change