def _get_interval_table(self, google_query, interval_field, params, dimensions=None, metrics=None, _cache_keys=None):
if interval_field != 'bm:quarter':
return google_query.get_table(params=params, dimensions=dimensions, metrics=metrics, _cache_keys=_cache_keys)
dimensions += [Column('ga:year')]
columns = google_query._columns_to_params(params.copy(), dimensions=dimensions, metrics=metrics)
result = Table(columns).new() # Decouple column refs
quarter_idx = result.column_to_index['bm:quarter']
if dimensions:
dimensions = [d for d in dimensions if not d.id.startswith('bm:quarter')]
# We assume that there are no date dimensions... Maybe not a safe assumption?
for (yr, q) in reversed(list(iter_quarters(params['start-date'], params['end-date']))):
start_date, end_date = quarter_to_dates(q, yr)
quarter_params = params.copy()
quarter_params.update({
'start-date': start_date,
'end-date': end_date,
})
quarter_params.pop('sort', None)
print(quarter_params)
t = google_query.get_table(params=quarter_params, dimensions=dimensions, metrics=metrics, renew=True, _cache_keys=_cache_keys)
quarter_str = "{}Q{}".format(yr, q)
for row in t.rows:
vals = row.values
vals.insert(quarter_idx, quarter_str)
result.add(vals)
return result
def test_rows(self):
def positive_int(n):
if not n or n < 0:
return
return int(n)
t = Table([
Column('foo', visible=1),
Column('bar', visible=0),
Column('baz', type_cast=positive_int, average=100),
])
data = [
(9999, '1', 123),
(123, '2', 1234),
(0000, '3', 23),
(123, '4', 123),
(123, '5', 0),
]
for d in data:
t.add(d)
self.assertEqual(len(t.rows), len(data))
self.assertEqual(t.get('foo').min_row, (None, None))
self.assertEqual(t.get('bar').max_row, (None, None))
self.assertEqual(t.get('baz').min_row[0], 23)
self.assertEqual(t.get('baz').max_row[0], 1234)
rows = t.iter_visible()
self.assertEqual(list(next(rows)), [t.columns[1], t.columns[0]])
self.assertEqual(list(next(rows)), ['1', 9999])
t.tag_rows()
self.assertFalse(t.rows[-1].tags)
def get_table(self, params, dimensions=None, metrics=None, renew=False, _cache_keys=None):
params = dict(params)
columns = self._columns_to_params(params, dimensions=dimensions, metrics=metrics)
t = Table(columns)
if renew:
t = t.new()
t._response_data = response_data = self._get_data(params)
if 'rows' not in response_data:
return t
for row in response_data['rows']:
t.add(row)
return t
def get_table(self,
params,
dimensions=None,
metrics=None,
_cache_keys=None):
columns = self._columns_to_params(params,
dimensions=dimensions,
metrics=metrics)
limit = min(self.num_rows, int(params.get('max-results', 10)))
t = Table(columns)
row_data = [self._stateful_cycle(col.id) for col in columns]
for _ in range(limit):
t.add(next(c) for c in row_data)
return t
def _get_geo(self, google_query, summary_metrics):
t = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': self.date_start,
'end-date': self.date_end,
'sort': '-ga:users',
'max-results': '50',
},
dimensions=[
Column('ga:country',
label='Country',
visible=1,
type_cast=_prune_abstract),
],
metrics=[col.new() for col in summary_metrics] + [
Column('ga:sessionsPerUser',
label='Sessions Per User',
threshold=0.0,
type_cast=float,
type_format=_format_float),
],
)
total = float(t.get('ga:users').sum)
out = Table(columns=[col.new() for col in t.columns] + [
Column(
'users', label='Users', visible=0,
type_format=_format_percent),
])
out.get('ga:users')._threshold = None
out.set_visible('users', 'ga:country')
idx_users = t.column_to_index['ga:users']
for row in t.rows[:5]:
out.add(row.values + [row.values[idx_users] * 100.0 / total])
out.tag_rows()
return out
def _get_social_search(self, google_query, date_start, date_end, summary_metrics, max_results=10):
organic_table = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': date_start,
'end-date': date_end,
'filters': 'ga:medium!=referral;ga:medium!=(not set);ga:socialNetwork==(not set)',
'sort': '-ga:pageviews',
'max-results': str(max_results),
},
dimensions=[
Column('ga:source', type_cast=_prune_abstract),
],
metrics=[col.new() for col in summary_metrics],
)
social_table = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': date_start,
'end-date': date_end,
'sort': '-ga:pageviews',
'max-results': str(max_results),
},
dimensions=[
Column('ga:socialNetwork', type_cast=_prune_abstract),
],
metrics=[col.new() for col in summary_metrics],
)
source_col = Column('source', label='Social & Search & Campaigns', visible=1, type_cast=_cast_title)
t = Table(columns=[
source_col,
] + [col.new() for col in summary_metrics])
for cells in social_table.iter_rows():
t.add(cells)
for cells in organic_table.iter_rows():
t.add(cells)
t.sort(reverse=True)
return t
def _get_versions(self, google_query, summary_metrics):
t = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': self.date_start,
'end-date': self.date_end,
'sort': '-ga:appVersion',
'max-results': '50',
},
dimensions=[
Column('ga:appVersion', label='Version', visible=1),
Column('ga:operatingSystem',
label='Operating System',
visible=2),
],
metrics=[col.new() for col in summary_metrics] + [],
)
t.set_visible('ga:users', 'ga:appVersion')
t.tag_rows()
col_exception = Column('ga:fatalExceptions',
label='Crashes',
type_cast=int,
type_format=h.human_int)
crashes_table = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': self.date_start,
'end-date': self.date_end,
'sort': '-ga:appVersion',
'max-results': '50',
},
dimensions=[
Column('ga:appVersion', label='Version', visible=1),
Column('ga:operatingSystem',
label='Operating System',
visible=2),
],
metrics=[col_exception],
)
crashes_lookup = dict(
((version, os), crashes)
for crashes, version, os in crashes_table.iter_rows(
'ga:fatalExceptions', 'ga:appVersion', 'ga:operatingSystem'))
total = float(t.get('ga:users').sum)
out = Table(columns=[
Column(
'users', label='Users', visible=0,
type_format=_format_percent),
Column('version', label='Version', visible=1),
])
latest_version = None
for i, (users, sessions, version, os) in enumerate(
t.iter_rows('ga:users', 'ga:sessions', 'ga:appVersion',
'ga:operatingSystem')):
row = out.add([users * 100.0 / total, u"%s on %s" % (version, os)])
row.tags = t.rows[i].tags
if not latest_version:
latest_version = version
crashes = crashes_lookup.get((version, os), 0)
if latest_version == version:
row.tag('latest')
if crashes and crashes > sessions * 0.1:
row.tag('crashes',
value=h.human_int(crashes),
is_positive=False,
is_prefixed=True)
self.latet_version = latest_version
out.limit(5)
out.sort(reverse=True)
return out
def _get_goals(self, google_query, interval_field):
goals_api = 'https://www.googleapis.com/analytics/v3/management/accounts/{accountId}/webproperties/{webPropertyId}/profiles/{profileId}/goals'
r = google_query.get(
goals_api.format(profileId=self.remote_data['id'],
**self.remote_data))
has_goals = r.get('items') or []
goal_metrics = [
Column('ga:goal{id}Completions'.format(id=g['id']),
label=g['name'],
type_cast=float) for g in has_goals if g.get('active')
]
if not goal_metrics:
return
# Note: max 10 metrics allowed
metrics = goal_metrics[-9:] + [Column('ga:sessions', type_cast=int)]
raw_table = google_query.get_table(
params={
'ids': 'ga:%s' % self.remote_id,
'start-date': self.previous_date_start, # Extra week
'end-date': self.date_end,
'sort': '-{}'.format(interval_field),
},
metrics=metrics,
dimensions=[
Column(interval_field),
],
)
if len(raw_table.rows) != 2:
# Less than 2 weeks of data available
return
t = Table(columns=[
Column('goal', label='Goals', visible=1, type_cast=_cast_title),
Column('completions',
label='Events',
visible=0,
type_cast=int,
type_format=h.human_int,
threshold=0),
])
num_sessions, num_sessions_last = [
next(v) for v in raw_table.iter_rows('ga:sessions')
]
this_week, last_week = raw_table.rows
col_compare = t.get('completions')
col_compare_delta = Column('%s:delta' % col_compare.id,
label='Events',
type_cast=float,
type_format=h.human_delta,
threshold=0)
has_completions = False
for col_id, pos in raw_table.column_to_index.items():
col = raw_table.columns[pos]
if not col.id.startswith('ga:goal'):
continue
completions, completions_last = this_week.values[
pos], last_week.values[pos]
percent_completions = completions * 100.0 / num_sessions if num_sessions else 0.0
percent_completions_last = completions_last * 100.0 / num_sessions_last if num_sessions_last else 0.0
row = t.add([col.label, completions])
if not row:
# Boring
continue
if completions > 0:
row.tag(type="Conversion",
value=_format_percent(percent_completions))
if completions + completions_last > 0:
has_completions = True
# Old method:
# delta = (percent_completions - percent_completions_last) / 100.0
# New method (same as GA shows):
delta = completions / completions_last - 1 if completions_last > 0.0 else 1.0
if abs(delta) > 0.001:
row.tag(type='delta',
value=delta,
column=col_compare_delta,
is_positive=delta > 0)
if not has_completions:
return
t.sort(reverse=True)
return t
def fetch(self, api_query):
last_month_date_start = self.date_end - datetime.timedelta(
days=self.date_end.day)
last_month_date_start -= datetime.timedelta(
days=last_month_date_start.day - 1)
# TODO: Check 'has_more'
week_params = {
'created[gte]': to_epoch(self.date_start),
'created[lt]':
to_epoch(self.date_end + datetime.timedelta(days=1)),
'limit': 100,
}
self.tables['customers'] = customers_table = Timeline()
items = api_query.get_paged('https://api.stripe.com/v1/customers',
params=week_params)
for item in items:
plan = (item.get('subscription') or {}).get('plan')
if plan:
plan = describe_plan(plan)
customers_table.add([
to_datetime(item['created']), ' '.join([
item.get('email') or '(no email)',
plan or '(no plan yet)',
])
])
##
self.tables['events'] = events_table = Timeline()
items = api_query.get_paged('https://api.stripe.com/v1/events',
params=week_params)
for item in items:
events_table.add([
to_datetime(item['created']),
describe_event(item),
])
##
historic_table = Table([
Column('created', visible=0),
Column('amount', label='Amount', visible=1),
])
items = api_query.get_paged('https://api.stripe.com/v1/charges',
params={
'created[gte]':
to_epoch(last_month_date_start),
'created[lt]':
to_epoch(self.date_end +
datetime.timedelta(days=1)),
'limit':
100,
})
for item in items:
if not item['paid'] or item['refunded']:
continue
historic_table.add([
to_datetime(item['created']),
item['amount'],
])
iter_historic = historic_table.iter_visible(reverse=True)
_, views_column = next(iter_historic)
monthly_data, max_value = sparse_cumulative(iter_historic,
final_date=self.date_end)
last_month, current_month = [[0], [0]]
if monthly_data:
last_month, current_month = monthly_data[-2:]
self.data['historic_data'] = encode_rows(monthly_data, max_value)
self.data['total_current'] = current_month[-1]
self.data['total_last'] = last_month[-1]
self.data['total_last_relative'] = last_month[
min(len(current_month), len(last_month)) - 1]
self.data['total_last_date_start'] = last_month_date_start
self.data['canary'] = None # XXX: Fix this before launching
def test_join(self):
t = Table([
Column('value'),
Column('joincol'),
Column('splitcol'),
])
expected = t.new()
t.add((1, 'foo', 'a'))
t.add((2, 'bar', 'a'))
t.add((3, 'baz', 'a'))
t.add((8, 'bar', 'b'))
t.add((7, 'baz', 'b'))
t.add((6, 'quux', 'b'))
expected.add((1, 'foo', 'a'))
expected.add((2, 'bar', 'a')).tag('Value', -6)
expected.add((3, 'baz', 'a')).tag('Value', -4)
split_table_delta(t, 'splitcol', 'joincol', 'value')
self.assertEqual(dump(t), dump(expected))