def _get_interval_table(self, google_query, interval_field, params, dimensions=None, metrics=None, _cache_keys=None): if interval_field != 'bm:quarter': return google_query.get_table(params=params, dimensions=dimensions, metrics=metrics, _cache_keys=_cache_keys) dimensions += [Column('ga:year')] columns = google_query._columns_to_params(params.copy(), dimensions=dimensions, metrics=metrics) result = Table(columns).new() # Decouple column refs quarter_idx = result.column_to_index['bm:quarter'] if dimensions: dimensions = [d for d in dimensions if not d.id.startswith('bm:quarter')] # We assume that there are no date dimensions... Maybe not a safe assumption? for (yr, q) in reversed(list(iter_quarters(params['start-date'], params['end-date']))): start_date, end_date = quarter_to_dates(q, yr) quarter_params = params.copy() quarter_params.update({ 'start-date': start_date, 'end-date': end_date, }) quarter_params.pop('sort', None) print(quarter_params) t = google_query.get_table(params=quarter_params, dimensions=dimensions, metrics=metrics, renew=True, _cache_keys=_cache_keys) quarter_str = "{}Q{}".format(yr, q) for row in t.rows: vals = row.values vals.insert(quarter_idx, quarter_str) result.add(vals) return result
def test_rows(self): def positive_int(n): if not n or n < 0: return return int(n) t = Table([ Column('foo', visible=1), Column('bar', visible=0), Column('baz', type_cast=positive_int, average=100), ]) data = [ (9999, '1', 123), (123, '2', 1234), (0000, '3', 23), (123, '4', 123), (123, '5', 0), ] for d in data: t.add(d) self.assertEqual(len(t.rows), len(data)) self.assertEqual(t.get('foo').min_row, (None, None)) self.assertEqual(t.get('bar').max_row, (None, None)) self.assertEqual(t.get('baz').min_row[0], 23) self.assertEqual(t.get('baz').max_row[0], 1234) rows = t.iter_visible() self.assertEqual(list(next(rows)), [t.columns[1], t.columns[0]]) self.assertEqual(list(next(rows)), ['1', 9999]) t.tag_rows() self.assertFalse(t.rows[-1].tags)
def get_table(self, params, dimensions=None, metrics=None, renew=False, _cache_keys=None): params = dict(params) columns = self._columns_to_params(params, dimensions=dimensions, metrics=metrics) t = Table(columns) if renew: t = t.new() t._response_data = response_data = self._get_data(params) if 'rows' not in response_data: return t for row in response_data['rows']: t.add(row) return t
def get_table(self, params, dimensions=None, metrics=None, _cache_keys=None): columns = self._columns_to_params(params, dimensions=dimensions, metrics=metrics) limit = min(self.num_rows, int(params.get('max-results', 10))) t = Table(columns) row_data = [self._stateful_cycle(col.id) for col in columns] for _ in range(limit): t.add(next(c) for c in row_data) return t
def _get_geo(self, google_query, summary_metrics): t = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': self.date_start, 'end-date': self.date_end, 'sort': '-ga:users', 'max-results': '50', }, dimensions=[ Column('ga:country', label='Country', visible=1, type_cast=_prune_abstract), ], metrics=[col.new() for col in summary_metrics] + [ Column('ga:sessionsPerUser', label='Sessions Per User', threshold=0.0, type_cast=float, type_format=_format_float), ], ) total = float(t.get('ga:users').sum) out = Table(columns=[col.new() for col in t.columns] + [ Column( 'users', label='Users', visible=0, type_format=_format_percent), ]) out.get('ga:users')._threshold = None out.set_visible('users', 'ga:country') idx_users = t.column_to_index['ga:users'] for row in t.rows[:5]: out.add(row.values + [row.values[idx_users] * 100.0 / total]) out.tag_rows() return out
def _get_social_search(self, google_query, date_start, date_end, summary_metrics, max_results=10): organic_table = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': date_start, 'end-date': date_end, 'filters': 'ga:medium!=referral;ga:medium!=(not set);ga:socialNetwork==(not set)', 'sort': '-ga:pageviews', 'max-results': str(max_results), }, dimensions=[ Column('ga:source', type_cast=_prune_abstract), ], metrics=[col.new() for col in summary_metrics], ) social_table = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': date_start, 'end-date': date_end, 'sort': '-ga:pageviews', 'max-results': str(max_results), }, dimensions=[ Column('ga:socialNetwork', type_cast=_prune_abstract), ], metrics=[col.new() for col in summary_metrics], ) source_col = Column('source', label='Social & Search & Campaigns', visible=1, type_cast=_cast_title) t = Table(columns=[ source_col, ] + [col.new() for col in summary_metrics]) for cells in social_table.iter_rows(): t.add(cells) for cells in organic_table.iter_rows(): t.add(cells) t.sort(reverse=True) return t
def _get_versions(self, google_query, summary_metrics): t = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': self.date_start, 'end-date': self.date_end, 'sort': '-ga:appVersion', 'max-results': '50', }, dimensions=[ Column('ga:appVersion', label='Version', visible=1), Column('ga:operatingSystem', label='Operating System', visible=2), ], metrics=[col.new() for col in summary_metrics] + [], ) t.set_visible('ga:users', 'ga:appVersion') t.tag_rows() col_exception = Column('ga:fatalExceptions', label='Crashes', type_cast=int, type_format=h.human_int) crashes_table = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': self.date_start, 'end-date': self.date_end, 'sort': '-ga:appVersion', 'max-results': '50', }, dimensions=[ Column('ga:appVersion', label='Version', visible=1), Column('ga:operatingSystem', label='Operating System', visible=2), ], metrics=[col_exception], ) crashes_lookup = dict( ((version, os), crashes) for crashes, version, os in crashes_table.iter_rows( 'ga:fatalExceptions', 'ga:appVersion', 'ga:operatingSystem')) total = float(t.get('ga:users').sum) out = Table(columns=[ Column( 'users', label='Users', visible=0, type_format=_format_percent), Column('version', label='Version', visible=1), ]) latest_version = None for i, (users, sessions, version, os) in enumerate( t.iter_rows('ga:users', 'ga:sessions', 'ga:appVersion', 'ga:operatingSystem')): row = out.add([users * 100.0 / total, u"%s on %s" % (version, os)]) row.tags = t.rows[i].tags if not latest_version: latest_version = version crashes = crashes_lookup.get((version, os), 0) if latest_version == version: row.tag('latest') if crashes and crashes > sessions * 0.1: row.tag('crashes', value=h.human_int(crashes), is_positive=False, is_prefixed=True) self.latet_version = latest_version out.limit(5) out.sort(reverse=True) return out
def _get_goals(self, google_query, interval_field): goals_api = 'https://www.googleapis.com/analytics/v3/management/accounts/{accountId}/webproperties/{webPropertyId}/profiles/{profileId}/goals' r = google_query.get( goals_api.format(profileId=self.remote_data['id'], **self.remote_data)) has_goals = r.get('items') or [] goal_metrics = [ Column('ga:goal{id}Completions'.format(id=g['id']), label=g['name'], type_cast=float) for g in has_goals if g.get('active') ] if not goal_metrics: return # Note: max 10 metrics allowed metrics = goal_metrics[-9:] + [Column('ga:sessions', type_cast=int)] raw_table = google_query.get_table( params={ 'ids': 'ga:%s' % self.remote_id, 'start-date': self.previous_date_start, # Extra week 'end-date': self.date_end, 'sort': '-{}'.format(interval_field), }, metrics=metrics, dimensions=[ Column(interval_field), ], ) if len(raw_table.rows) != 2: # Less than 2 weeks of data available return t = Table(columns=[ Column('goal', label='Goals', visible=1, type_cast=_cast_title), Column('completions', label='Events', visible=0, type_cast=int, type_format=h.human_int, threshold=0), ]) num_sessions, num_sessions_last = [ next(v) for v in raw_table.iter_rows('ga:sessions') ] this_week, last_week = raw_table.rows col_compare = t.get('completions') col_compare_delta = Column('%s:delta' % col_compare.id, label='Events', type_cast=float, type_format=h.human_delta, threshold=0) has_completions = False for col_id, pos in raw_table.column_to_index.items(): col = raw_table.columns[pos] if not col.id.startswith('ga:goal'): continue completions, completions_last = this_week.values[ pos], last_week.values[pos] percent_completions = completions * 100.0 / num_sessions if num_sessions else 0.0 percent_completions_last = completions_last * 100.0 / num_sessions_last if num_sessions_last else 0.0 row = t.add([col.label, completions]) if not row: # Boring continue if completions > 0: row.tag(type="Conversion", value=_format_percent(percent_completions)) if completions + completions_last > 0: has_completions = True # Old method: # delta = (percent_completions - percent_completions_last) / 100.0 # New method (same as GA shows): delta = completions / completions_last - 1 if completions_last > 0.0 else 1.0 if abs(delta) > 0.001: row.tag(type='delta', value=delta, column=col_compare_delta, is_positive=delta > 0) if not has_completions: return t.sort(reverse=True) return t
def fetch(self, api_query): last_month_date_start = self.date_end - datetime.timedelta( days=self.date_end.day) last_month_date_start -= datetime.timedelta( days=last_month_date_start.day - 1) # TODO: Check 'has_more' week_params = { 'created[gte]': to_epoch(self.date_start), 'created[lt]': to_epoch(self.date_end + datetime.timedelta(days=1)), 'limit': 100, } self.tables['customers'] = customers_table = Timeline() items = api_query.get_paged('https://api.stripe.com/v1/customers', params=week_params) for item in items: plan = (item.get('subscription') or {}).get('plan') if plan: plan = describe_plan(plan) customers_table.add([ to_datetime(item['created']), ' '.join([ item.get('email') or '(no email)', plan or '(no plan yet)', ]) ]) ## self.tables['events'] = events_table = Timeline() items = api_query.get_paged('https://api.stripe.com/v1/events', params=week_params) for item in items: events_table.add([ to_datetime(item['created']), describe_event(item), ]) ## historic_table = Table([ Column('created', visible=0), Column('amount', label='Amount', visible=1), ]) items = api_query.get_paged('https://api.stripe.com/v1/charges', params={ 'created[gte]': to_epoch(last_month_date_start), 'created[lt]': to_epoch(self.date_end + datetime.timedelta(days=1)), 'limit': 100, }) for item in items: if not item['paid'] or item['refunded']: continue historic_table.add([ to_datetime(item['created']), item['amount'], ]) iter_historic = historic_table.iter_visible(reverse=True) _, views_column = next(iter_historic) monthly_data, max_value = sparse_cumulative(iter_historic, final_date=self.date_end) last_month, current_month = [[0], [0]] if monthly_data: last_month, current_month = monthly_data[-2:] self.data['historic_data'] = encode_rows(monthly_data, max_value) self.data['total_current'] = current_month[-1] self.data['total_last'] = last_month[-1] self.data['total_last_relative'] = last_month[ min(len(current_month), len(last_month)) - 1] self.data['total_last_date_start'] = last_month_date_start self.data['canary'] = None # XXX: Fix this before launching
def test_join(self): t = Table([ Column('value'), Column('joincol'), Column('splitcol'), ]) expected = t.new() t.add((1, 'foo', 'a')) t.add((2, 'bar', 'a')) t.add((3, 'baz', 'a')) t.add((8, 'bar', 'b')) t.add((7, 'baz', 'b')) t.add((6, 'quux', 'b')) expected.add((1, 'foo', 'a')) expected.add((2, 'bar', 'a')).tag('Value', -6) expected.add((3, 'baz', 'a')).tag('Value', -4) split_table_delta(t, 'splitcol', 'joincol', 'value') self.assertEqual(dump(t), dump(expected))