Ejemplo n.º 1
0
    def summary_table(self):	
        data = {}
        for count, date in enumerate(self.date_list):
            totals = []
            metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
            for site in self.sites:
                rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]

                if rows[0]:
                    rows = self._remove_ga_names(rows)
                    rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
                
                    totals.extend(rows)
                else:
                    #print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
                    logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())

            
            aggregate  = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"])
            
            data[count] = aggregate      
        
        for period in data:
            data[period]["pv_per_session"] = data[period].get("pv_per_session", 0)/len(self.sites)
            data[period]["avg_session_time"] = (data[period].get("avg_session_time", 0)/len(self.sites))/60.0
            
            
        this_period = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous")
        this_period = utils.add_change(this_period, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly")

        return this_period          
Ejemplo n.º 2
0
 def social_network_table(self, num_articles):
     data = {}
     for count, date in enumerate(self.date_list):
         social = []
         metrics = "ga:pageviews,ga:users,ga:sessions"
         for site in self.sites:
             rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:socialNetwork", filters="ga:socialNetwork!=(not set)", 
                                             sort="-ga:users", aggregate_key="ga:socialNetwork")
             rows = self._remove_ga_names(rows)
             rows = utils.change_key_names(rows, {"social_network":"socialNetwork"})
             for row in rows:
                 row =  utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
                 
             social.extend(rows)
         
         aggregated = utils.aggregate_data(social, ["pageviews", "users", "sessions"], match_key="social_network")
         sorted = utils.sort_data(aggregated, "users", limit=15)
         data[count] = sorted
         
     added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions"], "previous", match_key="social_network")
     added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions"], "yearly", match_key="social_network")
     
     for row in added_change:
         filter = "ga:socialNetwork==%s" % row["social_network"]
         article = self.referral_articles(filter, num_articles)
         row["articles"] = article            
     
     return added_change
Ejemplo n.º 3
0
    def referral_articles(self, filter, limit):
        filters = config.ARTICLE_FILTER + ";" + filter
        article_previous = utils.StatsRange.get_previous_period(self.period, "DAILY")#how to do this
        data = {}
        for count, date in enumerate([self.period, article_previous]):
            articles = []
            for site in self.sites:
                rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:pageviews", dimensions="ga:pageTitle,ga:pagePath,ga:hostname", filters=filters, 
                                                sort="-ga:pageviews", aggregate_key="ga:pagePath")
                rows = self._remove_ga_names(rows)
                rows = utils.change_key_names(rows, {"title":"pageTitle", "path":"pagePath", "host":"hostname"})
                for row in rows:
                    path = row["path"]
                    title = row["title"]
                    new_path = self._remove_query_string(path)
                    new_title = self._get_title(path, title)
                    row["path"] = new_path
                    row["title"] = new_title
                    row["pageviews"] = float(row["pageviews"])
                
                articles.extend(rows)
            aggregated = utils.aggregate_data(articles, ["pageviews"], match_key="path")
            sorted = utils.sort_data(aggregated, "pageviews", limit=limit)
            data[count] = sorted
            #group

        added_change = utils.add_change(data[0], data[1], ["pageviews"], "previous", match_key="path")

        return added_change                
Ejemplo n.º 4
0
    def video_table(self):
        data = {}
        for count, date in enumerate([self.period]):
            table = []        
            for channel_num, channel in enumerate(self.channels):
                ids = self.channel_ids[channel]
                rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched,views", dimensions="video", filters=None, max_results="20", 
                                                sort="-estimatedMinutesWatched", aggregate_key="video")
                for row in rows:
                    row = utils.convert_to_floats(row, ["estimatedMinutesWatched", "views"])
                    video_name = analytics.get_video(row['video'])
                    try:
                        row['title'] = video_name['items'][0]['snippet']['title']
                    except IndexError:
                        row["title"] = "Video Not Found" 
                    row['channel'] = channel	

                table.extend(rows)
            		
            aggregated = utils.aggregate_data(table, ["estimatedMinutesWatched", "views"], match_key="video")
            sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=10)
            data[count] = sorted
            #group

        return data[0]       			
Ejemplo n.º 5
0
 def _get_by_source(self, subdivide_by_medium=False):
     data = {}
     aggregate_key = "ga:source"
     match_key = "source"
     if subdivide_by_medium:
         aggregate_key = "ga:sourceMedium"
         match_key = "source_medium"
     for count, date in enumerate(self.date_list):
         traffic_sources = []
         metrics = "ga:pageviews,ga:users"
         for site in self.sites:       
             rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=aggregate_key, sort="-ga:users", aggregate_key=aggregate_key)
             rows = self._remove_ga_names(rows)
             if subdivide_by_medium:
                 rows = utils.change_key_names(rows, {"source_medium":"sourceMedium"})
             for row in rows:
                 row =  utils.convert_to_floats(row, ["pageviews", "users"])
                 
             traffic_sources.extend(rows)
             
         aggregated = utils.aggregate_data(traffic_sources, ["pageviews", "users"], match_key=match_key)
         sorted = utils.sort_data(aggregated, "users")
         data[count] = sorted   
         
     added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key=match_key)
     added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key=match_key)
     
     return added_change
Ejemplo n.º 6
0
 def site_summary_table(self):
     #should this be combined with overall summary table?
     data = {}
     for count, date in enumerate(self.date_list):
         totals = []
         metrics="ga:pageviews,ga:users,ga:sessions,ga:pageviewsPerSession,ga:avgSessionDuration"
         for site in self.sites:
             rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics)]
             if rows[0]:
                 for row in rows:
                     row =  utils.convert_to_floats(row, metrics.split(","))
                     row["ga:site"] = site 
 
                 rows = self._remove_ga_names(rows)
                 rows = utils.change_key_names(rows, {"pv_per_session":"pageviewsPerSession", "avg_session_time":"avgSessionDuration"})
                                     
                 totals.extend(rows)
             else:
                 #print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
                 logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())
                                 
         aggregated = utils.aggregate_data(totals, ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], match_key="site")
         sorted = utils.sort_data(aggregated, "users")
         data[count] = sorted
         
     added_change = utils.add_change(data[0], data[1], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "previous", match_key="site")
     added_change = utils.add_change(added_change, data[2], ["pageviews", "users", "sessions", "pv_per_session", "avg_session_time"], "yearly", match_key="site")
     
     return added_change
Ejemplo n.º 7
0
    def traffic_source_table(self):
        data = {}
        source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
        traffic_source = []
        for source in source_types:
            row = {"insightTrafficSourceType":source, "source_total":0.0}
            traffic_source.append(row)
        
        for count, date in enumerate(self.date_list):
            table = []
            for channel in self.channels:
                ids = self.channel_ids[channel]
                rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None, 
                                                sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
                
                channel_total = 0.0
                for row in rows:
                    row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
                    row["channel"] = channel
                    channel_total += row["estimatedMinutesWatched"]                    
                    
                new_rows = []
                for source_type in traffic_source:
                    try:
                        result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
                        result["channel_total"] = channel_total
                        new_rows.append(result)
                        source_type["source_total"] += result["estimatedMinutesWatched"]
                    except KeyError:
                        new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})


                table.append(new_rows)

            
            new_table = []
            for channel in table:
                for row in channel:
                    #order each channel section by the total minutes watched for each source 
                    #calculate channels percentage breakdown by source
                    breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
                    row["source_percentage"] = breakdown
                    result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
                    row["source_total"] = result["source_total"]
                sort_row = utils.sort_data(channel, "source_total")  
                new_table.append(sort_row)
            
            #sort the new table of dictionaries by total channel watch time 

            sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
            data[count] = sorted_list       
            
            
        return data[0]            

            
Ejemplo n.º 8
0
    def traffic_source_table(self):
        data = {}
        source_types= ['ANNOTATION', 'EXT_URL', 'NO_LINK_OTHER', 'NOTIFICATION', 'PLAYLIST', 'RELATED_VIDEO', 'SUBSCRIBER', 'YT_CHANNEL', 'YT_OTHER_PAGE', 'YT_PLAYLIST_PAGE', 'YT_SEARCH']
        traffic_source = []
        for source in source_types:
            row = {"insightTrafficSourceType":source, "source_total":0.0}
            traffic_source.append(row)
        
        for count, date in enumerate(self.date_list):
            table = []
            for channel in self.channels:
                ids = self.channel_ids[channel]
                rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics="estimatedMinutesWatched", dimensions="insightTrafficSourceType", filters=None, 
                                                sort="-estimatedMinutesWatched", aggregate_key="insightTrafficSourceType")
                
                channel_total = 0.0
                for row in rows:
                    row = utils.convert_to_floats(row, ["estimatedMinutesWatched"])
                    row["channel"] = channel
                    channel_total += row["estimatedMinutesWatched"]                    
                    
                new_rows = []
                for source_type in traffic_source:
                    try:
                        result = utils.list_search(rows, "insightTrafficSourceType", source_type["insightTrafficSourceType"])
                        result["channel_total"] = channel_total
                        new_rows.append(result)
                        source_type["source_total"] += result["estimatedMinutesWatched"]
                    except KeyError:
                        new_rows.append({"insightTrafficSourceType":source_type["insightTrafficSourceType"], "channel":channel, "channel_total":channel_total, "estimatedMinutesWatched":0.0})


                table.append(new_rows)

            
            new_table = []
            for channel in table:
                for row in channel:
                    #order each channel section by the total minutes watched for each source 
                    #calculate channels percentage breakdown by source
                    breakdown = utils.percentage(row["estimatedMinutesWatched"], row["channel_total"])
                    row["source_percentage"] = breakdown
                    result = utils.list_search(traffic_source, "insightTrafficSourceType", row["insightTrafficSourceType"])
                    row["source_total"] = result["source_total"]
                sort_row = utils.sort_data(channel, "source_total")  
                new_table.append(sort_row)
            
            #sort the new table of dictionaries by total channel watch time 

            sorted_list = sorted(new_table, key = lambda k: k[0]["channel_total"], reverse = True)
            data[count] = sorted_list       
            
            
        return data[0]            
Ejemplo n.º 9
0
    def social_chart(self):

        current = self.period.start_date
        end = self.period.end_date
        dates = []
        while current <= end:
            current_range = utils.StatsRange("day", current, current)
            dates.append(current_range)
            current = current + timedelta(days=1)   
        

        data = {}
        network_data = {}
        for count, date in enumerate(dates):
            social = []
            network_social = []
            metrics = "ga:pageviews,ga:users,ga:sessions"
            for site in config.TABLES.keys():
                rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters="ga:socialNetwork!=(not set)", 
                                                sort="-ga:users")]
                if rows[0]:
                    rows = self._remove_ga_names(rows)
                    for row in rows:
                        row =  utils.convert_to_floats(row, ["pageviews", "users", "sessions"])
                        
                    if site in self.sites:    
                        social.extend(rows)
                        
                    network_social.extend(rows)
                else:
                    #print "No data for site " + site + " on " + date.get_start() + " - " + date.get_end()
                    logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())

            
            aggregate  = utils.aggregate_data(social, ["pageviews", "users", "sessions"])
            network_aggregate  = utils.aggregate_data(network_social, ["pageviews", "users", "sessions"])
            
            data[date.get_start()] = aggregate  
            network_data[date.get_start()] = network_aggregate  
            

        x_labels = []
        graph_data = {"users":[], "pageviews":[], "network_pageviews":[], "network_users":[]}
        for range in dates:
            x_labels.append(range.get_start())
            graph_data["users"].append(data[range.get_start()]["users"])
            graph_data["pageviews"].append(data[range.get_start()]["pageviews"])
            graph_data["network_users"].append(network_data[range.get_start()]["users"])
            graph_data["network_pageviews"].append(network_data[range.get_start()]["pageviews"])

        
        chart = utils.chart("Social Data", x_labels, graph_data, "Day", "Number")
        return chart
Ejemplo n.º 10
0
 def device_table(self):
     data = {}
     for count, date in enumerate(self.date_list):
         devices = []
         for site in self.sites:
             rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics="ga:users", dimensions="ga:deviceCategory", sort="-ga:users", aggregate_key="ga:deviceCategory")
             rows = self._remove_ga_names(rows)
             rows = utils.change_key_names(rows, {"device_category":"deviceCategory"})
             for row in rows:
                 row["users"] = float(row["users"])
                 
             devices.extend(rows)
         
         aggregated = utils.aggregate_data(devices, ["users"], match_key="device_category")
         sorted = utils.sort_data(aggregated, "users", limit=6)
         data[count] = sorted 
         
     added_change = utils.add_change(data[0], data[1], ["users"], "previous", match_key="device_category")
     added_change = utils.add_change(added_change, data[2], ["users"], "yearly", match_key="device_category")
     
     return added_change
Ejemplo n.º 11
0
    def country_table(self):
        countries = ["Czec", "Germa", "Denma", "Spai", "Franc", "Italy", 
            "Portug", "Swede", "Polan", "Brazi", "Belgiu", "Netherl", 
            "United Ki", "Irela", "United St", "Canad", "Austral", "New Ze"]
            
        countries_regex = "|".join(countries)
        filters = "ga:country=~%s" % countries_regex
        row_filters = "ga:country!~%s" % countries_regex
        data = {}
        for count, date in enumerate(self.date_list):
            breakdown = []
            metrics = "ga:pageviews,ga:users"
            for site in self.sites:
                rows = analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions="ga:country", filters=filters, 
                                                sort="-ga:pageviews", aggregate_key="ga:country")
                world_rows = [analytics.rollup_ids(self.site_ids[site], date.get_start(), date.get_end(), metrics=metrics, dimensions=None, filters=row_filters, 
                                                        sort="-ga:pageviews", aggregate_key=None)]

                if world_rows[0]:
                    world_rows[0]["ga:country"] = "ROW"
                else:
                    world_rows = [{"ga:country":"ROW", "ga:pageviews":0, "ga:users":0}]
                    
                rows.extend(world_rows)

                for row in rows:
                    row =  utils.convert_to_floats(row, metrics.split(","))
                    
                rows = self._remove_ga_names(rows)   
                breakdown.extend(rows)
                    
            aggregated = utils.aggregate_data(breakdown, ["pageviews", "users"], match_key="country")
            sorted = utils.sort_data(aggregated, "users")
            data[count] = sorted
            
        added_change = utils.add_change(data[0], data[1], ["pageviews", "users"], "previous", match_key="country")
        added_change = utils.add_change(added_change, data[2], ["pageviews", "users"], "yearly", match_key="country")
        
        return added_change
Ejemplo n.º 12
0
    def channel_summary_table(self):
        data = {}
        for count, date in enumerate(self.date_list):
            table = []
            metrics="subscribersGained,subscribersLost,estimatedMinutesWatched"
            for channel_num, channel in enumerate(self.channels):
                ids = self.channel_ids[channel]

                if count == 0:
                    subscriber_count = analytics.rollup_stats(ids)['subscriberCount'] #only gets the current subscriber count
                elif count == 1:
                    #last period date, work out last periods subscriber count from current periods sub change 
                    this_channel = utils.list_search(data[0], "channel", channel)
                    subscriber_count = this_channel["subscriberCount"] - this_channel["subscriberChange"]
                else:
                    #don't need to work out yearly sub change 
                    subscriber_count = 0.0
  
                rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics, dimensions=None, filters=None, sort=None, max_results=None, aggregate_key=None)]
                if rows[0]:
                    for row in rows:
                        row = utils.convert_to_floats(row, metrics.split(","))
                        row["channel"] = channel
                        row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
                        row["subscriberCount"] = float(subscriber_count)
                        
                    table.extend(rows)
                else:
                    print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
                    #logger.debug("No data for site " + site + " on " + date.get_start() + " - " + date.get_end())                    
                
            aggregated = utils.aggregate_data(table, ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], match_key= "channel")
            sorted = utils.sort_data(aggregated, "estimatedMinutesWatched")
            data[count] = sorted
            
        added_change = utils.add_change(data[0], data[1], ["subscriberChange", "subscriberCount", "estimatedMinutesWatched"], "previous", match_key= "channel")
        added_change = utils.add_change(added_change, data[2], ["subscriberChange", "estimatedMinutesWatched"], "yearly", match_key= "channel")
        
        return added_change
Ejemplo n.º 13
0
 def country_table(self):
     data = {}
     for count, date in enumerate(self.date_list):
         table = []
         metrics = "views,estimatedMinutesWatched,subscribersGained,subscribersLost"
         for channel in self.channels:
             ids = self.channel_ids[channel]
             rows = analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions="country", 
             												filters=None, max_results=None, sort="-estimatedMinutesWatched", aggregate_key="country")	
             for row in rows:
                 row =  utils.convert_to_floats(row, metrics.split(","))
                 row["subscriberChange"] = row["subscribersGained"] - row["subscribersLost"]
             table.extend(rows)
         	    			
         aggregated = utils.aggregate_data(table, (metrics + ",subscriberChange").split(","),  match_key="country")
         sorted = utils.sort_data(aggregated, "estimatedMinutesWatched", limit=20)
         data[count] = sorted
     
     added_change = utils.add_change(data[0], data[1], ["views","estimatedMinutesWatched","subscriberChange"], "previous", match_key="country")
     added_change = utils.add_change(added_change, data[2], ["views","estimatedMinutesWatched","subscriberChange"], "yearly", match_key="country")
     
     return added_change
Ejemplo n.º 14
0
 def channel_stats_table(self):
     data = {}
     for count, date in enumerate(self.date_list):
         table = []
         #just subscribersGained to get number of subscribers per 1000 views 
         metrics="views,likes,dislikes,comments,shares,subscribersGained"
         for channel_num, channel in enumerate(self.channels):
             ids = self.channel_ids[channel]
             rows = [analytics.rollup_ids(ids, date.get_start(), date.get_end(), metrics=metrics, dimensions=None)]
             if rows[0]:
                 for row in rows:
                     row = utils.convert_to_floats(row, metrics.split(","))
                     row["channel"] = channel
                     row["likeRate"] = utils.rate_per_1000(row["likes"], row['views'])
                     row["commentRate"] = utils.rate_per_1000(row["comments"], row['views'])
                     row["sharesRate"] = utils.rate_per_1000(row["shares"], row['views'])
                     row["subsRate"] = utils.rate_per_1000(row["subscribersGained"], row['views']) 
                     try:
                         row["likeRatio"] = utils.sig_fig(2, row["likes"] / row["dislikes"])
                     except ZeroDivisionError:
                         row["likeRatio"] = 0
                     try:
                         row["dislikeRatio"] = utils.sig_fig(2, row["dislikes"] / row["dislikes"])
                     except ZeroDivisionError:
                         row["dislikeRatio"] = 0                        
                 
                 table.extend(rows)
             else:
                 #print "No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end()
                 logger.debug("No data for channel " + channel + " on " + date.get_start() + " - " + date.get_end())     
                                            
         sorted = utils.sort_data(table, "views")
         data[count] = sorted
         
     added_change = utils.add_change(data[0], data[1], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "previous", match_key= "channel")
     added_change = utils.add_change(added_change, data[2], ["views", "likeRate", "commentRate", "sharesRate", "subsRate", "likeRatio", "dislikeRatio"], "yearly", match_key= "channel") 
     
     return added_change