def _get_by_source(self, subdivide_by_medium=False):
data = {}
aggregate_key = "ga:source"
match_key = "source"
if subdivide_by_medium:
aggregate_key = "ga:sourceMedium"
match_key = "source_medium"
for count, date in enumerate(self.date_list):
traffic_sources = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=aggregate_key, sort="-ga:users", aggregate_key=aggregate_key)
rows = self._remove_ga_names(rows)
if subdivide_by_medium:
rows = utils.change_key_names(rows, {"source_medium":"sourceMedium"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users"])
traffic_sources.extend(rows)
aggregated = utils.aggregate_data(traffic_sources, ["pageviews", "users"], match_key=match_key)
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key=match_key)
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key=match_key)
return added_change
def rollup_ids(self,
ids,
start,
end,
metrics,
dimensions=None,
filters=None,
sort=None,
max_results=None,
aggregate_key=None):
main_row = []
for id in ids:
if filters:
filter = filters + ";channel==%s" % id
else:
filter = "channel==%s" % id
results = self.run_analytics_report(start,
end,
metrics=metrics,
dimensions=dimensions,
filters=filter,
max_results=max_results,
sort=sort)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start +
" - " + end)
main_row = utils.aggregate_data(main_row, metrics.split(","),
aggregate_key)
return main_row
def social_network_table(self, num_articles):
data = {}
for count, date in enumerate(self.date_list):
social = []
metrics = "ga:pageviews,ga:users,ga:sessions"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:socialNetwork", filters="ga:socialNetwork!=(not set)",
sort="-ga:users", aggregate_key="ga:socialNetwork")
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"social_network":"socialNetwork"})
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
social.extend(rows)
aggregated = utils.aggregate_data(social, ["pageviews", "users", "sessions"], match_key="social_network")
sorted = utils.sort_data(aggregated, "users", limit=15)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions"], "previous", match_key="social_network")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions"], "yearly", match_key="social_network")
for row in added_change:
filter = "ga:socialNetwork==%s" % row["social_network"]
article = self.referral_articles(filter, num_articles)
row["articles"] = article
return added_change
def site_summary_table(self):
#should this be combined with overall summary table?
data = {}
for count, date in enumerate(self.date_list):
totals = []
metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
for site in self.sites:
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["ga:site"] = site
rows = self._remove_ga_names(rows)
rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
totals.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], match_key="site")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous", match_key="site")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly", match_key="site")
return added_change
def video_table(self):
data = {}
for count, date in enumerate([self.period]):
table = []
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched,views", dimensions="video", filters=None, max_results="20",
sort="-estimatedMinutesWatched", aggregate_key="video")
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched", "views"])
video_name = analytics.get_video(row['video'])
try:
row['title'] = video_name['items'][0]['snippet']['title']
except IndexError:
row["title"] = "Video Not Found"
row['channel'] = channel
table.extend(rows)
aggregated = utils.aggregate_data(table, ["estimatedMinutesWatched", "views"], match_key="video")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=10)
data[count] = sorted
#group
return data[0]
def traffic_source_table(self):
data = {}
source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
traffic_source = []
for source in source_types:
row = {"insightTrafficSourceType":source, "source_total":0.0}
traffic_source.append(row)
for count, date in enumerate(self.date_list):
table = []
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None,
sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
channel_total = 0.0
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
row["channel"] = channel
channel_total += row["estimatedMinutesWatched"]
new_rows = []
for source_type in traffic_source:
try:
result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
result["channel_total"] = channel_total
new_rows.append(result)
source_type["source_total"] += result["estimatedMinutesWatched"]
except KeyError:
new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})
table.append(new_rows)
new_table = []
for channel in table:
for row in channel:
#order each channel section by the total minutes watched for each source
#calculate channels percentage breakdown by source
breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
row["source_percentage"] = breakdown
result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
row["source_total"] = result["source_total"]
sort_row = utils.sort_data(channel, "source_total")
new_table.append(sort_row)
#sort the new table of dictionaries by total channel watch time
sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
data[count] = sorted_list
return data[0]
def traffic_source_table(self):
data = {}
source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
traffic_source = []
for source in source_types:
row = {"insightTrafficSourceType":source, "source_total":0.0}
traffic_source.append(row)
for count, date in enumerate(self.date_list):
table = []
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None,
sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
channel_total = 0.0
for row in rows:
row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
row["channel"] = channel
channel_total += row["estimatedMinutesWatched"]
new_rows = []
for source_type in traffic_source:
try:
result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
result["channel_total"] = channel_total
new_rows.append(result)
source_type["source_total"] += result["estimatedMinutesWatched"]
except KeyError:
new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})
table.append(new_rows)
new_table = []
for channel in table:
for row in channel:
#order each channel section by the total minutes watched for each source
#calculate channels percentage breakdown by source
breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
row["source_percentage"] = breakdown
result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
row["source_total"] = result["source_total"]
sort_row = utils.sort_data(channel, "source_total")
new_table.append(sort_row)
#sort the new table of dictionaries by total channel watch time
sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
data[count] = sorted_list
return data[0]
def social_chart(self):
current = self.period.start_date
end = self.period.end_date
dates = []
while current <= end:
current_range = utils.StatsRange("day", current, current)
dates.append(current_range)
current = current + timedelta(days=1)
data = {}
network_data = {}
for count, date in enumerate(dates):
social = []
network_social = []
metrics = "ga:pageviews,ga:users,ga:sessions"
for site in config.TABLES.keys():
rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters="ga:socialNetwork!=(not set)",
sort="-ga:users")]
if rows[0]:
rows = self._remove_ga_names(rows)
for row in rows:
row = utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
if site in self.sites:
social.extend(rows)
network_social.extend(rows)
else:
#print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregate = utils.aggregate_data(social, ["pageviews", "users", "sessions"])
network_aggregate = utils.aggregate_data(network_social, ["pageviews", "users", "sessions"])
data[date.get_start()] = aggregate
network_data[date.get_start()] = network_aggregate
x_labels = []
graph_data = {"users":[], "pageviews":[], "network_pageviews":[], "network_users":[]}
for range in dates:
x_labels.append(range.get_start())
graph_data["users"].append(data[range.get_start()]["users"])
graph_data["pageviews"].append(data[range.get_start()]["pageviews"])
graph_data["network_users"].append(network_data[range.get_start()]["users"])
graph_data["network_pageviews"].append(network_data[range.get_start()]["pageviews"])
chart = utils.chart("Social Data", x_labels, graph_data, "Day", "Number")
return chart
def rollup_stats(self, ids):
id_combo = ",".join(ids)
num_ids = len(ids)
results = self.get_stats(id_combo)
stats = {}
for row in results:
row["statistics"] = utils.convert_to_floats(
row["statistics"], row["statistics"].keys())
for key in row['statistics']:
try:
stats[key] += row["statistics"][key]
except:
stats[key] = row["statistics"][key]
return stats
def rollup_ids(self,
properties,
start,
end,
metrics,
dimensions=None,
filters=None,
sort=None,
max_results=None,
aggregate_key=None):
main_row = []
for property_details in properties:
id = property_details['id']
results = self.run_report(id,
start,
end,
metrics=metrics,
dimensions=dimensions,
filters=filters,
sort=sort,
max_results=max_results)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#no data
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start +
" - " + end)
str_dimensions = "None"
if dimensions:
str_dimensions = dimensions.encode('utf-8')
str_filters = "None"
if filters:
str_filters = filters.encode('utf-8')
logger.debug("for metrics: " + metrics + " dimensions: " +
str_dimensions + " filters: " + str_filters)
main_row = utils.aggregate_data(main_row, metrics.split(","),
aggregate_key)
return main_row
def country_table(self):
countries = ["Czec", "Germa", "Denma", "Spai", "Franc", "Italy",
"Portug", "Swede", "Polan", "Brazi", "Belgiu", "Netherl",
"United Ki", "Irela", "United St", "Canad", "Austral", "New Ze"]
countries_regex = "|".join(countries)
filters = "ga:country=~%s" % countries_regex
row_filters = "ga:country!~%s" % countries_regex
data = {}
for count, date in enumerate(self.date_list):
breakdown = []
metrics = "ga:pageviews,ga:users"
for site in self.sites:
rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:country", filters=filters,
sort="-ga:pageviews", aggregate_key="ga:country")
world_rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters=row_filters,
sort="-ga:pageviews", aggregate_key=None)]
if world_rows[0]:
world_rows[0]["ga:country"] = "ROW"
else:
world_rows = [{"ga:country":"ROW", "ga:pageviews":0, "ga:users":0}]
rows.extend(world_rows)
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
rows = self._remove_ga_names(rows)
breakdown.extend(rows)
aggregated = utils.aggregate_data(breakdown, ["pageviews", "users"], match_key="country")
sorted = utils.sort_data(aggregated, "users")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key="country")
return added_change
def channel_summary_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics="subscribersGained,subscribersLost,estimatedMinutesWatched"
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
if count == 0:
subscriber_count = analytics.rollup_stats(ids)['subscriberCount'] #only gets the current subscriber count
elif count == 1:
#last period date, work out last periods subscriber count from current periods sub change
this_channel = utils.list_search(data[0], "channel", channel)
subscriber_count = this_channel["subscriberCount"] - this_channel["subscriberChange"]
else:
#don't need to work out yearly sub change
subscriber_count = 0.0
rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["channel"] = channel
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
row["subscriberCount"] = float(subscriber_count)
table.extend(rows)
else:
print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
#logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
aggregated = utils.aggregate_data(table, ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], match_key= "channel")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], "previous", match_key= "channel")
added_change = utils.add_change(added_change, data[2], ["subscriberChange", "estimatedMinutesWatched"], "yearly", match_key= "channel")
return added_change
def country_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
metrics = "views,estimatedMinutesWatched,subscribersGained,subscribersLost"
for channel in self.channels:
ids = self.channel_ids[channel]
rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions="country",
filters=None, max_results=None, sort="-estimatedMinutesWatched", aggregate_key="country")
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
table.extend(rows)
aggregated = utils.aggregate_data(table, (metrics + ",subscriberChange").split(","), match_key="country")
sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=20)
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["views","estimatedMinutesWatched","subscriberChange"], "previous", match_key="country")
added_change = utils.add_change(added_change, data[2], ["views","estimatedMinutesWatched","subscriberChange"], "yearly", match_key="country")
return added_change
def channel_stats_table(self):
data = {}
for count, date in enumerate(self.date_list):
table = []
#just subscribersGained to get number of subscribers per 1000 views
metrics="views,likes,dislikes,comments,shares,subscribersGained"
for channel_num, channel in enumerate(self.channels):
ids = self.channel_ids[channel]
rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions=None)]
if rows[0]:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
row["channel"] = channel
row["likeRate"] = utils.rate_per_1000(row["likes"], row['views'])
row["commentRate"] = utils.rate_per_1000(row["comments"], row['views'])
row["sharesRate"] = utils.rate_per_1000(row["shares"], row['views'])
row["subsRate"] = utils.rate_per_1000(row["subscribersGained"], row['views'])
try:
row["likeRatio"] = utils.sig_fig(2, row["likes"] / row["dislikes"])
except ZeroDivisionError:
row["likeRatio"] = 0
try:
row["dislikeRatio"] = utils.sig_fig(2, row["dislikes"] / row["dislikes"])
except ZeroDivisionError:
row["dislikeRatio"] = 0
table.extend(rows)
else:
#print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
logger.debug("No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end())
sorted = utils.sort_data(table, "views")
data[count] = sorted
added_change = utils.add_change(data[0], data[1], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "previous", match_key= "channel")
added_change = utils.add_change(added_change, data[2], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "yearly", match_key= "channel")
return added_change
def rollup_ids(self, properties, start, end, metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None):
main_row = []
for property_details in properties:
id = property_details['id']
results = self.run_report(id, start, end, metrics=metrics, dimensions=dimensions, filters=filters, sort=sort, max_results=max_results)
rows = utils.format_data_rows(results)
if rows:
for row in rows:
row = utils.convert_to_floats(row, metrics.split(","))
main_row.extend(rows)
else:
#no data
#print "No data for id " + id + " " + " on " + start + " - " + end
logger.debug("No data for " + id + " " + " on " + start + " - " + end)
str_dimensions = "None"
if dimensions:
str_dimensions = dimensions.encode('utf-8')
str_filters = "None"
if filters:
str_filters = filters.encode('utf-8')
logger.debug("for metrics: " + metrics + " dimensions: " + str_dimensions + " filters: " + str_filters)
main_row = utils.aggregate_data(main_row, metrics.split(","), aggregate_key)
return main_row