def zranked(cls, pk, parent_dimension='_', metric='hits', period=None,
at=None, start=0, size=10, sort_dir=None, tzoffset=None):
period, ats, tzoffset = Period.get_days(period, at)
dt = ats or [Period.convert(cls.now(), tzoffset)]
return map(try_loads,
_ranked(cls.whale_driver(), pk, parent_dimension, metric,
period, dt, start, size, sort_dir=sort_dir))
def total(cls, pk, metric, dimension='_', period=None, at=None, index=None,
tzoffset=None):
period, ats, tzoffset = Period.get_days(period, at, tzoffset=tzoffset)
top, bot = parse_formula(metric)
dimension = maybe_dumps(dimension)
if not ats and not index:
index = -1
if isinstance(index, int):
pps = cls.plotpoints(pk, dimension, metric, period=period, points_type=list)
return pps[dimension][metric][index][1]
else:
if not bot:
pps = cls.plotpoints(pk, dimension, metric, period=period,
tzoffset=tzoffset)
ppsm = pps[dimension][metric]
return sum([ppsm[dt] for dt in ats if dt in ppsm])
else:
top_pps = cls.plotpoints(pk, dimension, top, period=period,
tzoffset=tzoffset)
bot_pps = cls.plotpoints(pk, dimension, bot, period=period,
tzoffset=tzoffset)
top_ppsm = top_pps[dimension][top]
bot_ppsm = bot_pps[dimension][bot]
top_tot = sum([top_ppsm[dt] for dt in ats if dt in top_ppsm])
bot_tot = sum([bot_ppsm[dt] for dt in ats if dt in bot_ppsm])
return bot_tot and top_tot/bot_tot or 0
def totals(cls, pk, dimensions=None, metrics=None, periods=None, at=None):
if not periods:
periods = DEFAULT_PERIODS
if not isinstance(periods, list):
periods = [periods]
metrics = metrics or ['hits']
if not isinstance(metrics, list):
metrics = [metrics]
ratios = []
for metric in metrics:
if '/' in metric:
metrics.remove(metric)
ratios.append(metric)
metrics += metric.split('/')
d = {}
for p in periods:
period, ats, tzoffset = Period.get_days(p,at)
p_data = cls.plotpoints(pk, dimensions, metrics, period=p,at=at)
p_totals = dict()
for dim in p_data.keys():
p_totals[dim] = dict()
for met, vals in p_data[dim].items():
p_totals[dim][met] = sum([
v for k, v in vals.items()
if k in ats])
for rat in ratios:
top, bot = parse_formula(rat)
topt, bott = p_totals[dim][top], p_totals[dim][bot]
p_totals[dim][rat] = bott and topt / bott or 0
d[str(p)] = p_totals
return d
def totals(cls, pk, dimensions=None, metrics=None, periods=None, at=None):
if not periods:
periods = DEFAULT_PERIODS
if not isinstance(periods, list):
periods = [periods]
metrics = metrics or ['hits']
if not isinstance(metrics, list):
metrics = [metrics]
ratios = []
for metric in metrics:
if '/' in metric:
metrics.remove(metric)
ratios.append(metric)
metrics += metric.split('/')
d = {}
for p in periods:
period, ats, tzoffset = Period.get_days(p, at)
p_data = cls.plotpoints(pk, dimensions, metrics, period=p, at=at)
p_totals = dict()
for dim in p_data.keys():
p_totals[dim] = dict()
for met, vals in p_data[dim].items():
p_totals[dim][met] = sum(
[v for k, v in vals.items() if k in ats])
for rat in ratios:
top, bot = parse_formula(rat)
topt, bott = p_totals[dim][top], p_totals[dim][bot]
p_totals[dim][rat] = bott and topt / bott or 0
d[str(p)] = p_totals
return d
def rank_subdimensions_ratio(cls,
pk,
numerator,
denominator='hits',
dimension='_',
period=None,
recursive=True,
points=False):
top, bottom = numerator, denominator
p_obj, ats, tzoffset = Period.get_days(period)
p_s = str(p_obj)
d_k = keyify(dimension)
top_points = cls.totals(pk, dimension, top, periods=[p_s])
top_total = p_s in top_points and top_points[p_s][d_k][top] or 0
bottom_points = cls.totals(pk, dimension, bottom, periods=[p_s])
bottom_total = p_s in bottom_points and bottom_points[p_s][d_k][
bottom] or 0
ratio_total = bottom_total and float(top_total / bottom_total) or 0
ranked = dict()
def info(sub):
pps = cls.plotpoints(
pk, sub, [top, bottom, '%s/%s' % (top, bottom)],
period=period)[sub]
top_pps = pps[top]
bottom_pps = pps[bottom]
sub_top_sum = sum({k: v
for k, v in top_pps.items()
if k in ats}.values())
sub_bottom_sum = sum(
{k: v
for k, v in bottom_pps.items() if k in ats}.values())
ratio = sub_bottom_sum and float(sub_top_sum / sub_bottom_sum) or 0
difference = ratio_total and (ratio -
ratio_total) / ratio_total or 0
important = sub_bottom_sum > 5 and math.fabs(difference) > .1
data = {
'points': pps,
'score': ratio,
'difference': difference,
'effect': difference * sub_bottom_sum * ratio_total,
'value': sub_top_sum,
'count': sub_bottom_sum,
'important': important,
'dimension': sub
}
if not points:
del data['points']
return data
_subs = recursive and cls.all_subdimensions or cls.get_subdimensions
for sub in map(maybe_dumps, _subs(pk, dimension)):
ranked[sub] = info(sub)
return ranked
def total(cls,
pk,
metric,
dimension='_',
period=None,
at=None,
index=None,
tzoffset=None):
period, ats, tzoffset = Period.get_days(period, at, tzoffset=None)
top, bot = parse_formula(metric)
if not ats and not index:
index = -1
if isinstance(index, int):
pps = cls.plotpoints(pk,
dimension,
metric,
period=period,
points_type=list)
return pps[dimension][metric][index][1]
else:
if not bot:
pps = cls.plotpoints(pk, dimension, metric, period=period)
ppsm = pps[dimension][metric]
return sum([ppsm[dt] for dt in ats if dt in ppsm])
else:
top_pps = cls.plotpoints(pk, dimension, top, period=period)
bot_pps = cls.plotpoints(pk, dimension, bot, period=period)
top_ppsm = top_pps[dimension][top]
bot_ppsm = bot_pps[dimension][bot]
top_tot = sum([top_ppsm[dt] for dt in ats if dt in top_ppsm])
bot_tot = sum([bot_ppsm[dt] for dt in ats if dt in bot_ppsm])
return bot_tot and top_tot / bot_tot or 0
def render_divs(cls,
pk,
metric,
dimension='_',
period=None,
at=None,
tzoffset=None,
format=None,
hidden=False):
period, ats, tzoffset = Period.get_days(period, at, tzoffset=tzoffset)
top, bot = parse_formula(metric)
pps = cls.plotpoints(pk, dimension, metric, period=period)
ppsm = pps[dimension][metric]
if not format:
if bot:
format = 'pct'
else:
format = 'grouped'
def fmt(v):
import locale
if v == 'None':
v = None
f = format
if f == 'int':
f = lambda s: int(float(s or 0))
elif f == 'float':
v = v or 0.0
f = float
elif f == 'grouped':
v = v or 0
f = lambda s: locale.format('%d', int(float(s)), True)
elif f in ['pct', 'percent', '%', 'ratio']:
v = min(101, v and float(v) * 100 or 0)
f = lambda s: '%.2f%%' % s
elif f in ['cash', 'money', 'usd', '$', 'dollars', 'cents']:
v = v and float(v) or 0.0
if f == 'cents':
v = v / 100.0
f = locale.currency
if not f:
return v
return callable(f) and f(v) or v
hidden = hidden and 'style="display: none"' or ''
rep = lambda s: s.format(
pk=pk, metric=metric, dimension=dimension, hidden=hidden)
table = rep(
'<table {hidden} data-hw-pk="{pk}" data-hw-name="{{name}}" \
data-hw-dimension="{dimension}" data-metric="{metric}">'
) + '\n'.join([
'<tr><td>%s</td><td>%s</td></tr>' %
(at.replace(' 00:00:00', ''), fmt(count))
for at, count in ppsm.items()
]) + '</table>'
return table
def rank_subdimensions_ratio(cls, pk, numerator, denominator='hits',
dimension='_', period=None, recursive=True, points=False):
top, bottom = numerator, denominator
p_obj, ats, tzoffset = Period.get_days(period)
p_s = str(p_obj)
d_k = keyify(dimension)
top_points = cls.totals(pk, dimension, top,
periods=[p_s])
top_total = p_s in top_points and top_points[p_s][d_k][top] or 0
bottom_points = cls.totals(pk, dimension, bottom,
periods=[p_s])
bottom_total = p_s in bottom_points and bottom_points[p_s][d_k][bottom] or 0
ratio_total = bottom_total and float(top_total / bottom_total) or 0
ranked = dict()
def info(sub):
pps = cls.plotpoints(pk, sub, [top, bottom, '%s/%s' % (top, bottom)], period=period)[sub]
top_pps = pps[top]
bottom_pps = pps[bottom]
sub_top_sum = sum({k: v for k, v in top_pps.items() if k in ats}.values())
sub_bottom_sum = sum({k: v for k, v in bottom_pps.items() if k in ats}.values())
ratio = sub_bottom_sum and float(sub_top_sum / sub_bottom_sum) or 0
difference = ratio_total and (ratio - ratio_total) / ratio_total or 0
important = sub_bottom_sum > 5 and math.fabs(difference) > .1
data = {
'points': pps,
'score': ratio,
'difference': difference,
'effect': difference * sub_bottom_sum * ratio_total,
'value': sub_top_sum,
'count': sub_bottom_sum,
'important': important,
'dimension': sub
}
if not points:
del data['points']
return data
_subs = recursive and cls.all_subdimensions or cls.get_subdimensions
for sub in map(maybe_dumps, _subs(pk, dimension)):
ranked[sub] = info(sub)
return ranked
def rank_subdimensions_scalar(cls,
pk,
dimension='_',
metric='hits',
period=None,
recursive=True,
prune_parents=True,
points=False):
p_obj, ats, tzoffset = Period.get_days(period)
p_s = str(period)
d_k = keyify(dimension)
# sum of all values in metric
total = cls.totals(pk, dimension, metric,
periods=[p_s])[p_s][d_k][metric]
ranked = dict()
def info(sub):
pps = cls.plotpoints(pk, sub, metric, period=period)[sub][metric]
sub_total = sum(pps.values())
data = {
'points': pps,
'score': sub_total,
'important': sub_total > 10 and (sub_total > (total / 10)),
'effect': total - sub_total,
'difference': total - sub_total,
'value': sub_total,
'count': sub_total,
'dimension': sub
}
if not points:
del data['points']
return data
_subs = recursive and cls.all_subdimensions or cls.get_subdimensions
for sub in map(maybe_dumps, _subs(pk, dimension)):
ranked[sub] = info(sub)
# Prune parents
if recursive and prune_parents:
for sub, info in ranked.items():
children = map(maybe_dumps, cls.get_subdimensions(pk, sub))
children_total = sum(
map(lambda s: ranked[s]['score'], children))
if info['important'] and (info['score'] -
children_total) < (total / 10):
info['important'] = False
return ranked
def render_hw_plotpoint_table(cls, pk, metric, dimension='_', period=None, at=None,
tzoffset=None, format=None, hidden=False, graph_color=''):
period, ats, tzoffset = Period.get_days(period, tzoffset=tzoffset)
top, bot = parse_formula(metric)
pps = cls.plotpoints(pk, dimension, metric, period=period,
tzoffset=tzoffset)
ppsm = pps[dimension][metric]
if not format:
if bot:
format = 'pct'
else:
format = 'grouped'
def fmt(v):
import locale
if v == 'None':
v = None
f = format
if f == 'int':
f = lambda s: int(float(s or 0))
elif f == 'float':
v = v or 0.0
f = float
elif f == 'grouped':
v = v or 0
f = lambda s: locale.format('%d', int(float(s)), True)
elif f in ['pct', 'percent', '%', 'ratio']:
v = min(101, v and float(v)*100 or 0)
f = lambda s: '%.2f%%'%s
elif f in ['cash', 'money', 'usd', '$', 'dollars', 'cents']:
v = v and float(v) or 0.0
if f == 'cents':
v = v/100.0
f = locale.currency
if not f:
return v
return callable(f) and f(v) or v
hidden = hidden and 'style="display: none"' or ''
rep = lambda s: s.format(pk=pk, metric=metric, dimension=dimension,
hidden=hidden, color=graph_color)
table = rep('<table {hidden} data-hw-pk="{pk}" data-hw-name="{{name}}" \
data-hw-dimension="{dimension}" data-metric="{metric}" \
data-hw-color="{color}">')+'\n'.join([
'<tr><td>%s</td><td>%s</td></tr>'%(at.replace(' 00:00:00', ''), fmt(count) )
for at, count in ppsm.items()])+'</table>'
return table
def zranked(cls,
pk,
parent_dimension='_',
metric='hits',
period=None,
at=None,
start=0,
size=10,
sort_dir=None,
tzoffset=None):
period, ats, tzoffset = Period.get_days(period, at)
dt = ats or [Period.convert(cls.now(), tzoffset)]
return map(
try_loads,
_ranked(cls.whale_driver(),
pk,
parent_dimension,
metric,
period,
dt,
start,
size,
sort_dir=sort_dir))
def rank_subdimensions_scalar(cls, pk, dimension='_', metric='hits',
period=None, recursive=True, prune_parents=True, points=False):
p_obj, ats, tzoffset = Period.get_days(period)
p_s = str(period)
d_k = keyify(dimension)
# sum of all values in metric
total = cls.totals(pk, dimension, metric, periods=[p_s])[p_s][d_k][metric]
ranked = dict()
def info(sub):
pps = cls.plotpoints(pk, sub, metric, period=period)[sub][metric]
sub_total = sum(pps.values())
data = {
'points': pps,
'score': sub_total,
'important': sub_total > 10 and (sub_total > (total / 10)),
'effect': total - sub_total,
'difference': total - sub_total,
'value': sub_total,
'count': sub_total,
'dimension': sub
}
if not points:
del data['points']
return data
_subs = recursive and cls.all_subdimensions or cls.get_subdimensions
for sub in map(maybe_dumps, _subs(pk, dimension)):
ranked[sub] = info(sub)
# Prune parents
if recursive and prune_parents:
for sub, info in ranked.items():
children = map(maybe_dumps, cls.get_subdimensions(pk, sub))
children_total = sum(map(lambda s: ranked[s]['score'], children))
if info['important'] and (info['score'] - children_total) < (total / 10):
info['important'] = False
return ranked
def scalar_plotpoints(cls, pk, dimensions=None, metrics=None, at=None,
depth=0, period=None, flot_time=False, points_type=OrderedDict):
metrics = metrics or ['hits']
at = at or times.now()
if isinstance(metrics, basestring):
metrics = [metrics]
p_obj, ats, tzoffset = Period.get_days(period,at)
p_s = str(p_obj)
dts = list(p_obj.datetimes_strs(end=Period.parse_dt_str(at)))
sparse = _retrieve(cls.whale_driver(), pk, dimensions, metrics, period=p_obj)
nonsparse = defaultdict(dict)
if flot_time:
points_type = list
for dim, mets in sparse.items():
for met, points in mets.items():
nonsparse[dim][met] = []
use_method = False
met_name = met
if ':' in met:
met_name, use_method = met.split(':')
const_value = False
if met_name in TIME_MATRIX:
const_value = float(p_obj.getUnits()[0] /
TIME_MATRIX[met_name])
# Try to parse static metrics too
elif met_name == '_count':
const_value = len(dts)
try:
const_value = float(met_name)
except:
pass
last_value = total = 0
for dt in dts:
dt_obj = Period.parse_dt_str(dt)
if met_name == '_days_in_month':
from calendar import monthrange
const_value = monthrange(dt_obj.year, dt_obj.month)[1]
if flot_time:
dt_t = to_flot_time(dt_obj)
else:
dt_t = dt
if const_value:
value = const_value
else:
value = points[dt] if dt in points else 0
if use_method == 'count' or not use_method:
value = value
elif use_method in ['+', 'sum', 'add', 'cumulative']:
total += value
value = total
elif use_method in ['_', 'set', 'last', 'level']:
if not last_value:
last_value = value
if not value:
value = last_value
last_value = value
nonsparse[dim][met].append([dt_t, float(value)])
nonsparse[dim][met] = points_type(nonsparse[dim][met])
if depth > 0:
for sub in cls.get_subdimensions(pk, dimensions):
nonsparse = dict(nonsparse.items() +
cls.plotpoints(pk, sub, metrics, at=at, depth=depth - 1, period=period,
flot_time=flot_time, points_type=points_type).items())
return nonsparse
def scalar_plotpoints(cls,
pk,
dimensions=None,
metrics=None,
at=None,
depth=0,
period=None,
flot_time=False,
points_type=OrderedDict):
metrics = metrics or ['hits']
at = at or times.now()
if isinstance(metrics, basestring):
metrics = [metrics]
p_obj, ats, tzoffset = Period.get_days(period, at)
p_s = str(p_obj)
dts = list(p_obj.datetimes_strs(end=Period.parse_dt_str(at)))
sparse = _retrieve(cls.whale_driver(),
pk,
dimensions,
metrics,
period=p_obj)
nonsparse = defaultdict(dict)
if flot_time:
points_type = list
for dim, mets in sparse.items():
for met, points in mets.items():
nonsparse[dim][met] = []
use_method = False
met_name = met
if ':' in met:
met_name, use_method = met.split(':')
const_value = False
if met_name in TIME_MATRIX:
const_value = float(p_obj.getUnits()[0] /
TIME_MATRIX[met_name])
# Try to parse static metrics too
elif met_name == '_count':
const_value = len(dts)
try:
const_value = float(met_name)
except:
pass
last_value = total = 0
for dt in dts:
dt_obj = Period.parse_dt_str(dt)
if met_name == '_days_in_month':
from calendar import monthrange
const_value = monthrange(dt_obj.year, dt_obj.month)[1]
if flot_time:
dt_t = to_flot_time(dt_obj)
else:
dt_t = dt
if const_value:
value = const_value
else:
value = points[dt] if dt in points else 0
if use_method == 'count' or not use_method:
value = value
elif use_method in ['+', 'sum', 'add', 'cumulative']:
total += value
value = total
elif use_method in ['_', 'set', 'last', 'level']:
if not last_value:
last_value = value
if not value:
value = last_value
last_value = value
nonsparse[dim][met].append([dt_t, float(value)])
nonsparse[dim][met] = points_type(nonsparse[dim][met])
if depth > 0:
for sub in cls.get_subdimensions(pk, dimensions):
nonsparse = dict(
nonsparse.items() +
cls.plotpoints(pk,
sub,
metrics,
at=at,
depth=depth - 1,
period=period,
flot_time=flot_time,
points_type=points_type).items())
return nonsparse
