def __init__(self, group_strategy, timeseries_unit = 'day', date_filter_attributes=None):
self.filterable_attributes = [DATE, CUSTOM_ATTRIBUTE, TRANSFER_ACCOUNT, USER]
self.timeseries_unit = timeseries_unit
self.date_filter_attributes = date_filter_attributes
self.metrics = []
# Special case query-- this is just used to calculate grouped per_user
if group_strategy:
total_users_grouped_timeseries_query = db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]))
self.total_users_grouped_timeseries = metric.Metric(
metric_name='total_population_grouped',
query=group_strategy.build_query_group_by_with_join(total_users_grouped_timeseries_query, User),
object_model=User,
stock_filters=[],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[ADD_MISSING_DAYS_TO_TODAY, ACCUMULATE_TIMESERIES])
# Special case query-- this is just used to calculate ungrouped per_user
total_users_timeseries_query = db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date')).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]))
self.total_users_timeseries = metric.Metric(
metric_name='total_population',
query=total_users_timeseries_query,
object_model=User,
stock_filters=[],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[ADD_MISSING_DAYS_TO_TODAY, ACCUMULATE_TIMESERIES])
def get_cash_flows():
date_range_filter_schema = DateRangeFilterSchema().load(request.args)
if date_range_filter_schema.errors:
return {'errors': date_range_filter_schema.errors}, 400
cash_flow_schema = CashFlowSchema()
amounts = db.session.query(
func.sum(Record.amount).label("cash_flow"),
func.sum(
case([(Record.record_type == Record.RECORD_TYPE_INCOME, Record.amount)], else_=0)
).label('income'),
func.sum(
case([(Record.record_type == Record.RECORD_TYPE_EXPENSE, Record.amount)], else_=0)
).label('expense'),
func.date_trunc('month', Record.date).label("date"),
).group_by(
func.date_trunc('month', Record.date)
).order_by(
func.date_trunc('month', Record.date)
)
if 'date_from' in date_range_filter_schema.data:
amounts = amounts.filter(Record.date >= date_range_filter_schema.data['date_from'])
if 'date_to' in date_range_filter_schema.data:
amounts = amounts.filter(Record.date < date_range_filter_schema.data['date_to'])
return {'objects': cash_flow_schema.dump(amounts, many=True).data}
def _investments(self):
today_investments = Investment.query.filter(
cast(Investment.added_at, Date) == date.today(),
Investment.status.in_(
(get_enum('INVESTMENT_PENDING'),
get_enum('INVESTMENT_SUCCESSED'))
)
).order_by("added_at desc").limit(10)
history_investments = db_session.query(
func.date_trunc('day', Investment.added_at),
func.sum(Investment.amount)).group_by(
func.date_trunc('day', Investment.added_at)
).order_by(func.date_trunc('day', Investment.added_at)).all()
total_investments = db_session.query(
func.sum(Investment.amount)).scalar()
today_invest_amount = db_session.query(
func.sum(Investment.amount)).filter(
cast(Investment.added_at, Date) == date.today(),
Investment.status.in_(
(get_enum('INVESTMENT_PENDING'),
get_enum('INVESTMENT_SUCCESSED'))
)
).scalar()
if not today_invest_amount:
today_invest_amount = 0
app.jinja_env.globals['today_invest_amount'] = today_invest_amount
app.jinja_env.globals['today_investments'] = today_investments
app.jinja_env.globals['total_investments'] = total_investments
app.jinja_env.globals['history_investments'] = history_investments
def author(mit_id, conn):
"""
Returns an author object for insertion into mongo summary collection.
The format is as follows:
{"_id": {"name": <name>, "mitid": <mitid>},
"type": "author",
"size": <num docs>,
"downloads": <num downloads>,
"countries": [
{"country": <3 ltr code>, "downloads": <num downloads>},...
]
"dates": [
{"date": <YYYY-MM-DD>, "downloads": <num>},...
]}
"""
requests_to_authors = requests.join(documents)\
.join(documents_authors)\
.join(authors)
totals = select([
authors.c.mit_id,
authors.c.name,
select([func.count()])
.select_from(documents_authors.join(authors))
.where(authors.c.mit_id==bindparam('mit_id'))
.label('size'),
select([func.count()])
.select_from(requests_to_authors)
.where(authors.c.mit_id==bindparam('mit_id'))
.label('downloads')
])\
.where(authors.c.mit_id==bindparam('mit_id'))
countries = select([requests.c.country, func.count().label('downloads')])\
.select_from(requests_to_authors)\
.where(authors.c.mit_id==bindparam('mit_id'))\
.group_by(requests.c.country)
dates = select([
func.date_trunc('day', requests.c.datetime).label('date'),
func.count().label('downloads')])\
.select_from(requests_to_authors)\
.where(authors.c.mit_id==bindparam('mit_id'))\
.group_by(func.date_trunc('day', requests.c.datetime))
author_obj = {'type': 'author'}
res = conn.execute(totals, mit_id=mit_id).first()
author_obj['_id'] = {'name': res['name'], 'mitid': res['mit_id']}
author_obj['size'] = res['size']
author_obj['downloads'] = res['downloads']
res = conn.execute(countries, mit_id=mit_id)
for row in res:
author_obj.setdefault('countries', [])\
.append({'country': row['country'], 'downloads': row['downloads']})
res = conn.execute(dates, mit_id=mit_id)
for row in res:
author_obj.setdefault('dates', [])\
.append({'date': row['date'].strftime('%Y-%m-%d'),
'downloads': row['downloads']})
return author_obj
def make_filter(cls, field, ftype, value):
filter = None
if ftype == 'IN':
filter = field.in_([v for v in value if v])
elif ftype == 'date_gt':
filter = field > value
elif ftype == 'date_gte':
filter = field >= value
elif ftype == 'date_gt_now_less':
qty, granularity = value.split(" ")
filter = field > func.date_trunc(granularity, func.now() - cast(value, Interval()))
elif ftype == 'date_lt_now_less':
qty, granularity = value.split(" ")
filter = field < func.date_trunc(granularity, func.now() - cast(value, Interval()))
elif ftype == 'date_x_last_n':
qty, granularity, count_current_period = value.split(" ")
filter = (field > func.date_trunc(granularity, func.now() - cast("%s %s" % (qty, granularity), Interval())), field < func.date_trunc(granularity, func.now() - cast('0', Interval())),)
if count_current_period == 'on':
filter = filter[0]
elif ftype == 'date_month_ne':
filter = extract('month', field) != value
elif ftype == 'date_month_gt':
filter = extract('month', field) > value
elif ftype == 'date_month_lt':
filter = extract('month', field) < value
elif ftype == 'date_month_eq':
filter = extract('month', field) == value
elif ftype == 'date_hour_ne':
filter = extract('hour', field) != value
elif ftype == 'date_hour_gt':
filter = extract('hour', field) > value
elif ftype == 'date_hour_lt':
filter = extract('hour', field) < value
elif ftype == 'date_hour_eq':
filter = extract('hour', field) == value
elif ftype == 'date_lt':
filter = field < value
elif ftype == 'date_lte':
filter = field <= value
elif ftype == '=':
filter = field == value
elif ftype == '!=':
filter = field != value
elif ftype == '>':
filter = field > value
elif ftype == '>=':
filter = field >= value
elif ftype == '<':
filter = field < value
elif ftype == '<=':
filter = field <= value
elif ftype == 'like':
filter = field.ilike(value)
return filter
def get_balance(year, month):
(_, day) = calendar.monthrange(year, month)
start_date = datetime.date(year, month, 1)
end_date = datetime.date(year, month, day)
balance_schema = BalanceSchema()
amounts = db.session.query(
func.sum(Record.amount).label("cash_flow"),
func.sum(
case([(Record.record_type == Record.RECORD_TYPE_INCOME, Record.amount)], else_=0)
).label('income'),
func.sum(
case([(Record.record_type == Record.RECORD_TYPE_EXPENSE, Record.amount)], else_=0)
).label('expense'),
func.date_trunc('month', Record.date).label("date"),
).filter(
func.extract('year', Record.date) == year,
func.extract('month', Record.date) == month,
).group_by(
func.date_trunc('month', Record.date)
).first()
current_balance = db.session.query(
func.sum(
case([(Record.date < start_date, Record.amount)], else_=0)
).label('start_balance'),
func.sum(Record.amount).label("end_balance")
).filter(
Record.date <= end_date
).first()
if amounts:
balance = balance_schema.dump({
'cash_flow': amounts.cash_flow,
'income': amounts.income,
'expense': amounts.expense,
'date': amounts.date,
'start_balance': current_balance.start_balance,
'end_balance': current_balance.end_balance,
}).data
else:
balance = balance_schema.dump({
'cash_flow': 0,
'income': 0,
'expense': 0,
'date': end_date,
'start_balance': current_balance.start_balance,
'end_balance': current_balance.end_balance,
}).data
return balance
def get_items_per_minute():
items_per_minute = OrderItem.query.with_entities(func.date_trunc('minute', OrderItem.created_at),
func.count(OrderItem.id)).group_by(
func.date_trunc('minute', OrderItem.created_at)).order_by(
func.date_trunc('minute', OrderItem.created_at)) \
.limit(15) \
.all()
data_items_per_minute = []
labels_items_per_minute = []
for item in items_per_minute:
labels_items_per_minute.append(item[0].strftime("%H:%M:%S"))
data_items_per_minute.append(item[1])
legend_items_per_minute = 'Total received'
labels_items_per_minute = labels_items_per_minute
values_items_per_minute = data_items_per_minute
return labels_items_per_minute, legend_items_per_minute, values_items_per_minute
def get_orders_per_minute():
orders_per_minute = CssOrder.query.with_entities(func.date_trunc('minute', CssOrder.created_at),
func.count(CssOrder.id)).group_by(
func.date_trunc('minute', CssOrder.created_at)).order_by(
func.date_trunc('minute', CssOrder.created_at)) \
.limit(15) \
.all()
data_orders_per_minute = []
labels_orders_per_minute = []
for order in orders_per_minute:
labels_orders_per_minute.append(order[0].strftime("%H:%M:%S"))
data_orders_per_minute.append(order[1])
legend_orders_per_minute = 'Total received'
labels_orders_per_minute = labels_orders_per_minute
values_orders_per_minute = data_orders_per_minute
return labels_orders_per_minute, legend_orders_per_minute, values_orders_per_minute
def get_orders_per_minute_complete():
orders_per_minute_complete = CssOrder.query \
.with_entities(func.date_trunc('minute', CssOrder.completed_at), func.count(CssOrder.id)) \
.filter(CssOrder.status == CssConstants.ORDER_COMPLETE) \
.group_by(func.date_trunc('minute', CssOrder.completed_at)) \
.order_by(func.date_trunc('minute', CssOrder.completed_at)) \
.limit(15) \
.all()
data_orders_per_minute_complete = []
labels_orders_per_minute_complete = []
for order in orders_per_minute_complete:
labels_orders_per_minute_complete.append(order[0].strftime("%H:%M:%S"))
data_orders_per_minute_complete.append(order[1])
legend_orders_per_minute_complete = 'Completed'
labels_orders_per_minute_complete = labels_orders_per_minute_complete
values_orders_per_minute_complete = data_orders_per_minute_complete
return labels_orders_per_minute_complete, legend_orders_per_minute_complete, values_orders_per_minute_complete
def get_items_per_minute_complete():
items_per_minute_complete = OrderItem.query \
.with_entities(func.date_trunc('minute', OrderItem.completed_at), func.count(OrderItem.id)) \
.filter(OrderItem.status == CssConstants.ORDER_COMPLETE) \
.group_by(func.date_trunc('minute', OrderItem.completed_at)) \
.order_by(func.date_trunc('minute', OrderItem.completed_at)) \
.limit(15) \
.all()
data_items_per_minute_complete = []
labels_items_per_minute_complete = []
for item in items_per_minute_complete:
labels_items_per_minute_complete.append(item[0].strftime("%H:%M:%S"))
data_items_per_minute_complete.append(item[1])
legend_items_per_minute_complete = 'Completed'
labels_items_per_minute_complete = labels_items_per_minute_complete
values_items_per_minute_complete = data_items_per_minute_complete
return labels_items_per_minute_complete, legend_items_per_minute_complete, values_items_per_minute_complete
def groupby_created(precision='month'):
conn = sa.connect()
month = func.date_trunc('month', users.c.created).label('month')
q = select([month,
func.count(users.c.id)]).group_by('month').order_by('month')
#return conn.execute(q).fetchall()
return [(dt.strftime('%b %Y'), num)
for (dt, num) in conn.execute(q).fetchall()]
def dlc(dlc_id, conn):
requests_to_dlcs = requests.join(documents)\
.join(documents_dlcs)\
.join(dlcs)
totals = select([
dlcs.c.canonical_name,
dlcs.c.display_name,
select([func.count()])
.select_from(documents_dlcs.join(dlcs))
.where(dlcs.c.id==bindparam('dlc_id'))
.label('size'),
select([func.count()])
.select_from(requests_to_dlcs)
.where(dlcs.c.id==bindparam('dlc_id'))
.label('downloads')
])\
.where(dlcs.c.id==bindparam('dlc_id'))
countries = select([requests.c.country, func.count().label('downloads')])\
.select_from(requests_to_dlcs)\
.where(dlcs.c.id==bindparam('dlc_id'))\
.group_by(requests.c.country)
dates = select([
func.date_trunc('day', requests.c.datetime).label('date'),
func.count().label('downloads')])\
.select_from(requests_to_dlcs)\
.where(dlcs.c.id==bindparam('dlc_id'))\
.group_by(func.date_trunc('day', requests.c.datetime))
dlc_obj = {'type': 'dlc'}
res = conn.execute(totals, dlc_id=dlc_id).first()
dlc_obj['_id'] = {'canonical': res['canonical_name'],
'display': res['display_name']}
dlc_obj['size'] = res['size']
dlc_obj['downloads'] = res['downloads']
res = conn.execute(countries, dlc_id=dlc_id)
for row in res:
dlc_obj.setdefault('countries', [])\
.append({'country': row['country'],
'downloads': row['downloads']})
res = conn.execute(dates, dlc_id=dlc_id)
for row in res:
dlc_obj.setdefault('dates', [])\
.append({'date': row['date'].strftime('%Y-%m-%d'),
'downloads': row['downloads']})
return dlc_obj
async def count_for_period(conn, params):
period = params.get('period')
if not period:
label_edge = 'year'
else:
label_edge = {
'year': 'month',
'month': 'day',
'day': 'hour'
}[period['kind']]
label = func.date_trunc(label_edge, play.c.date).label('label')
stmt = select([label, func.count().label('value')])
if period:
stmt = stmt.where(
func.date_trunc(period['kind'], play.c.date) == period['value'])
stmt = stmt.group_by(label).order_by(label)
filter_kind = params.get('filter_kind')
if filter_kind == 'artist':
filter_column = artist.c.id
from_clause = play.join(track).join(album).join(artist)
elif filter_kind == 'album':
filter_column = album.c.id
from_clause = play.join(track).join(album)
elif filter_kind == 'track':
filter_column = track.c.id
from_clause = play.join(track)
else:
filter_column = None
from_clause = None
if filter_column is not None:
filter_value = params.get('filter_value')
if not filter_value:
raise ValidationError(
{'filter_value': ['This field is required.']})
stmt = stmt.where(filter_column == filter_value)
if from_clause is not None:
stmt = stmt.select_from(from_clause)
result = await conn.execute(stmt)
return await result.fetchall()
def handle(doc_id, conn):
totals = select([
documents.c.title,
documents.c.handle,
select([func.count()])
.select_from(requests)
.where(requests.c.document_id==bindparam('doc_id'))
.label('downloads')
])\
.where(documents.c.id==bindparam('doc_id'))
parents = select([authors.c.name, authors.c.mit_id])\
.select_from(authors.join(documents_authors).join(documents))\
.where(documents.c.id==bindparam('doc_id'))
countries = select([requests.c.country, func.count().label('downloads')])\
.where(requests.c.document_id==bindparam('doc_id'))\
.group_by(requests.c.country)
dates = select([
func.date_trunc('day', requests.c.datetime).label('date'),
func.count().label('downloads')])\
.where(requests.c.document_id==bindparam('doc_id'))\
.group_by(func.date_trunc('day', requests.c.datetime))
handle_obj = {'type': 'handle'}
res = conn.execute(totals, doc_id=doc_id).first()
handle_obj['_id'] = res['handle']
handle_obj['title'] = res['title']
handle_obj['downloads'] = res['downloads']
res = conn.execute(parents, doc_id=doc_id)
for row in res:
handle_obj.setdefault('parents', [])\
.append({'mitid': row['mit_id'], 'name': row['name']})
res = conn.execute(countries, doc_id=doc_id)
for row in res:
handle_obj.setdefault('countries', [])\
.append({'country': row['country'],
'downloads': row['downloads']})
res = conn.execute(dates, doc_id=doc_id)
for row in res:
handle_obj.setdefault('dates', [])\
.append({'date': row['date'].strftime('%Y-%m-%d'),
'downloads': row['downloads']})
return handle_obj
def get_avg_time_to_completion():
avg_wait_times_per_minute = OrderItem.query \
.with_entities(func.date_trunc('minute', OrderItem.created_at),
func.avg(OrderItem.completed_at - OrderItem.created_at).label('average')) \
.filter(OrderItem.status == CssConstants.ORDER_COMPLETE) \
.group_by(func.date_trunc('minute', OrderItem.created_at)) \
.order_by(func.date_trunc('minute', OrderItem.created_at)) \
.limit(15) \
.all()
data_avg_wait_times = []
labels_avg_wait_times = []
for wait_time in avg_wait_times_per_minute:
seconds = wait_time[1].seconds % 60
minutes = round((wait_time[1].seconds / 60) % 60, 2)
hours = (wait_time[1].seconds // 3600) % 24
days = wait_time[1].days
labels_avg_wait_times.append(wait_time[0].strftime("%H:%M:%S"))
data_avg_wait_times.append(minutes)
legend_avg_wait_times = 'Avg. Time To Completion'
labels_avg_wait_times = labels_avg_wait_times
values_avg_wait_times = data_avg_wait_times
return labels_avg_wait_times, legend_avg_wait_times, values_avg_wait_times
def overall(conn):
totals = select([
select([func.count()]).select_from(requests).label('downloads'),
select([func.count()]).select_from(documents).label('size')])
countries = select([requests.c.country, func.count().label('downloads')])\
.group_by(requests.c.country)
dates = select([func.date_trunc('day', requests.c.datetime).label('date'),
func.count().label('downloads')])\
.group_by(func.date_trunc('day', requests.c.datetime))
overall_obj = {'type': 'overall'}
res = conn.execute(totals).first()
overall_obj['downloads'] = res['downloads']
overall_obj['size'] = res['size']
res = conn.execute(countries)
for row in res:
overall_obj.setdefault('countries', [])\
.append({'country': row['country'],
'downloads': row['downloads']})
res = conn.execute(dates)
for row in res:
overall_obj.setdefault('dates', [])\
.append({'date': row['date'].strftime('%Y-%m-%d'),
'downloads': row['downloads']})
return overall_obj
def Stats(self, request, context):
with session_scope() as session:
date_ = func.date_trunc("day", Report.time_received)
stats = (session.query(date_, func.sum(StrandInReport.state)).join(
StrandInReport,
StrandInReport.report_id == Report.report_id).filter(
StrandInReport.strand_id == request.strand_id).group_by(
date_).all())
return sb_pb2.StatsRes(
strand_id=1,
times=[timestamp_from_datetime(datetime.datetime.now())],
total_incubating_strands=[6],
total_infected_strands=[3],
total_removed_strands=[2],
)
def get_build_history(project, end_period, days=90):
def date_to_key(dt):
return int(dt.replace(
minute=0, hour=0, second=0, microsecond=0
).strftime('%s'))
query = db.session.query(
Job.result,
func.count('*').label('num'),
func.date_trunc(literal('day'), Job.date_created).label('date')
).filter(
Job.project_id == project.id,
Job.date_created >= end_period - timedelta(days=days),
Job.date_created < end_period,
Job.status == Status.finished,
Source.revision_sha != None, # NOQA
Source.patch_id == None,
).group_by(Job.result, 'date').order_by('date asc')
# group results by day
results = {}
for n in xrange(days):
results[date_to_key(end_period - timedelta(days=n))] = {
'counts': {
'passed': 0,
'failed': 0,
'aborted': 0,
'unknown': 0,
},
'numBuilds': 0,
}
for result, num, date in query:
this = results[date_to_key(date)]
this['counts'][result.name] += num
this['numBuilds'] += num
return results
def make_grouping(cls, logger, grouping_info):
group_type, group_args, group_name = grouping_info
grouping = None
if group_type == "extract":
subfield, field_name = group_args
real_field = getattr(cls, field_name, None)
if real_field:
if subfield == 'ampm':
grouping = case(whens = [(cast(extract('hour', real_field), Integer()) > 12, 1),], else_ = 0).label(group_name)
else:
grouping = cast(extract(subfield, real_field), Integer()).label(group_name)
else:
logger.error("Invalid grouping %s (%s)", grouping_info, cls)
elif group_type == "date_trunc":
subfield, field_name = group_args
real_field = getattr(cls, field_name, None)
if real_field:
grouping = func.date_trunc(subfield, real_field)
grouping = grouping.label(group_name)
else:
logger.error("Invalid grouping %s (%s)", grouping_info, cls)
elif group_type == "func":
logger.error("Grouping by func not implemented yet")
elif group_type == 'coalesce_trim':
# trim(coalesce(field_name, ''))
field_name = group_args.get('field', group_name)
real_field = getattr(cls, field_name, None)
if real_field:
grouping = func.coalesce(real_field, group_args.get('coalesce_to', ''))
if group_args.get('trim', True):
grouping = func.trim(grouping)
grouping = grouping.label(group_name)
else:
logger.error("Invalid grouping %s (%s)", grouping_info, cls)
else:
logger.error("Unknown grouping type %s", group_type)
return grouping
def __init__(self,
group_strategy,
timeseries_unit='day',
date_filter_attributes=None):
self.filterable_attributes = [
DATE, CUSTOM_ATTRIBUTE, TRANSFER_ACCOUNT, USER
]
self.timeseries_unit = timeseries_unit
self.date_filter_attributes = date_filter_attributes
self.metrics = []
# Timeseries Metrics
if group_strategy:
users_created_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date'), group_strategy.group_by_column)\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User])), User)
aggregated_users_created_query = group_strategy\
.build_query_group_by_with_join(db.session.query(func.count(User.id).label('volume'), group_strategy.group_by_column), User)
else:
users_created_timeseries_query = db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date'))\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]))
aggregated_users_created_query = None
total_users_created_query = db.session.query(
func.count(User.id).label('volume'))
self.metrics.append(
metric.Metric(
metric_name='users_created',
is_timeseries=True,
query=users_created_timeseries_query,
aggregated_query=aggregated_users_created_query,
total_query=total_users_created_query,
object_model=User,
#stock_filters=[filters.beneficiary_filters], # NOTE: Do we still want this filter?
stock_filters=[],
query_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
aggregated_query_caching_combinatory_strategy=metrics_cache.
SUM_OBJECTS,
total_query_caching_combinatory_strategy=metrics_cache.TALLY,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
))
if group_strategy:
active_users_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date'), group_strategy.group_by_column)\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer])), CreditTransfer)
aggregated_active_users_query = group_strategy.build_query_group_by_with_join(
db.session.query(
func.count(func.distinct(
CreditTransfer.sender_user_id)).label('volume'),
group_strategy.group_by_column), CreditTransfer)
else:
active_users_timeseries_query = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date'))\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]))
aggregated_active_users_query = None
total_active_users_query = db.session.query(
func.count(func.distinct(
CreditTransfer.sender_user_id)).label('volume'))
self.metrics.append(
metric.Metric(
metric_name='active_users',
is_timeseries=True,
query=active_users_timeseries_query,
aggregated_query=aggregated_active_users_query,
total_query=total_active_users_query,
object_model=CreditTransfer,
#stock_filters=[filters.beneficiary_filters], # NOTE: Do we still want this filter?
stock_filters=[],
query_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
aggregated_query_caching_combinatory_strategy=metrics_cache.
QUERY_ALL,
total_query_caching_combinatory_strategy=metrics_cache.
QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
))
if group_strategy:
total_users_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date'), group_strategy.group_by_column)\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User])), User)
else:
total_users_timeseries_query = db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]).label('date'))\
.group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[User]))
self.metrics.append(
metric.Metric(
metric_name='total_population_cumulative',
is_timeseries=True,
query=total_users_timeseries_query,
total_query=total_users_created_query,
aggregated_query=aggregated_active_users_query,
object_model=User,
stock_filters=[],
query_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
aggregated_query_caching_combinatory_strategy=metrics_cache.
SUM_OBJECTS,
total_query_caching_combinatory_strategy=metrics_cache.TALLY,
filterable_by=self.filterable_attributes,
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
query_actions=[
ADD_MISSING_DAYS_TO_TODAY, ACCUMULATE_TIMESERIES,
FORMAT_TIMESERIES
]))
except (QBException,),e:
self.logger.error("Can't build query with bad parameters : %s", e)
return
# Prepare the query and the specific stuff depending on the 'whattodo'
filter_nbs, grouping_nbs = self.prepare_filters_and_grouping()
filter_nbs = self.generate_prefilters(filter_nbs)
self.prepare_query()
# Prepare and generate the filters and grouping
self.generate_filters(filter_nbs)
self.generate_groupings(grouping_nbs)
# Decide of the correct granularity when displaying evolutions
if self.granularity_criteria is None and self.query_parameters.get("whattodo", "count") in ('evolution', 'event_delta_evolution'):
date_delta = self.end_date - self.start_date
self.granularity = get_granularity(date_delta, self.MIN_POINTS, self.MAX_POINTS)#'month'
self.granularity_criteria = func.date_trunc(self.granularity, self.date_criteria_field).label("Date")
# A specific granularity has speen specified or decided, generate
# a subquery with generate_series to avoid holes in the time
if self.granularity_criteria is not None:
self.subquery = self.session.query(func.generate_series(func.date_trunc(self.granularity, cast(self.start_date, DateTime())), func.date_trunc(self.granularity, cast(self.end_date, DateTime())), cast('1 %s' % self.granularity, Interval())).label('Temps')).subquery()
self.select.insert(1,self.granularity_criteria)
self.display_translations.insert(1, None)
self.group_by.insert(0,self.granularity_criteria)
### Extras ###
# The user specified a boundary
extras = self.query_parameters.get("extras", {})
if extras and 'boundary' in self.extras:
boundary_max = extras.get('boundary', {}).get('max', None)
classes_case = []
if boundary_max and self.field_to_color is not None:
classes_case.append((self.field_to_color > boundary_max, literal_column("'breaks_max_bound'",String)))
def get(self, repo: Repository):
"""
Return various stats per-day for the given repository.
"""
stat = request.args.get("stat")
if not stat:
return self.error({"stat": "invalid stat"})
if stat not in STAT_CHOICES:
return self.error({"stat": "invalid stat"})
aggregate = request.args.get("aggregate", "time")
if aggregate not in ("time", "build"):
return self.error({"aggregate": "invalid aggregate"})
branch = request.args.get("branch")
since = request.args.get("since")
if since:
date_end = datetime.utcfromtimestamp(
float(since)).replace(tzinfo=timezone.utc)
else:
date_end = timezone.now() + timedelta(days=1)
date_end = date_end.replace(minute=0, second=0, microsecond=0)
if aggregate == "time":
resolution = request.args.get("resolution", "1d")
points = int(
request.args.get("points") or POINTS_DEFAULT[resolution])
if resolution == "1h":
grouper = func.date_trunc("hour", Build.date_created)
decr_res = decr_hour
elif resolution == "1d":
grouper = func.date_trunc("day", Build.date_created)
date_end = date_end.replace(hour=0)
decr_res = decr_day
elif resolution == "1w":
grouper = func.date_trunc("week", Build.date_created)
date_end = date_end.replace(hour=0)
date_end -= timedelta(days=date_end.weekday())
decr_res = decr_week
elif resolution == "1m":
grouper = func.date_trunc("month", Build.date_created)
date_end = date_end.replace(hour=0, day=1)
decr_res = decr_month
elif aggregate == "build":
grouper = Build.number
points = int(request.args.get("points") or 100)
queryset = build_queryset(repo.id, stat, grouper)
if aggregate == "time":
date_begin = date_end
for _ in range(points):
date_begin = decr_res(date_begin)
queryset = queryset.filter(Build.date_created >= date_begin,
Build.date_created < date_end)
elif aggregate == "build":
revision_shas = get_revisions(repo, branch, limit=points * 2)
queryset = (queryset.join(
Source, Source.id == Build.source_id).filter(
Source.revision_sha.in_(revision_shas)).order_by(
Build.number.desc()))
queryset = queryset.limit(points)
if aggregate == "time":
results = {
# HACK(dcramer): force (but dont convert) the timezone to be utc
# while this isnt correct, we're not looking for correctness yet
k.replace(tzinfo=timezone.utc): v
for k, v in queryset
}
data = []
cur_date = date_end
for _ in range(points):
cur_date = decr_res(cur_date)
data.append({
"time":
int(float(cur_date.strftime("%s.%f")) * 1000),
"value": (int(float(results[cur_date]))
if results.get(cur_date) else
(0 if stat in ZERO_FILLERS else None)),
})
elif aggregate == "build":
data = [{
"build":
k,
"value": (int(float(v)) if v is not None else
(0 if stat in ZERO_FILLERS else None)),
} for k, v in sorted(queryset, key=lambda x: -x[0])]
return self.respond(data)
def __call__(
self,
session: Session = Depends(get_db),
averaging_window: AveragingWindow = Query(
None,
alias="averaging-window",
description=
"Bucket and average samples within a particular time window.",
),
start_time: datetime = Query(
...,
alias="start-time",
description=
"ISO 8601 format string representing the start time of the range of data requested.",
example="2020-02-01T17:28:14.723333",
),
end_time: datetime = Query(
...,
alias="end-time",
description=
"ISO 8601 format string representing the end time of the range of data requested.",
example="2020-02-01T17:28:14.723333",
),
):
if start_time > end_time:
raise HTTPException(status_code=422,
detail="Start time is later than end time")
if averaging_window is not None:
window = func.date_trunc(
averaging_window.value,
self.package.values_table.timestamp).label("timestamp")
raw_data = (session.query(
window,
*[
func.avg(getattr(self.package.values_table,
field)).label(field)
for field in self.package.values_table.attrs()
],
).filter(window > start_time).filter(
window < end_time).group_by(window).order_by(window).all())
else:
raw_data = (session.query(self.package.values_table).filter(
self.package.values_table.timestamp > start_time).filter(
self.package.values_table.timestamp < end_time).all())
if self.data_type == DataType.raw:
return raw_data
metadata = {
row.uid: row
for row in session.query(self.package.metadata_table).all()
}
selected_sensors = [
uid for uid, sensor in metadata.items()
if sensor.type == self.data_type.value
]
return [{
"timestamp": row.timestamp,
**{(metadata[uid].name or uid): Calculations[str(self.package)][self.data_type](uid, row, metadata)
for uid in selected_sensors},
} for row in raw_data]
def __init__(self, group_strategy, timeseries_unit = 'day'):
self.filterable_attributes = [DATE, CUSTOM_ATTRIBUTE, TRANSFER_ACCOUNT, USER]
self.timeseries_unit = timeseries_unit
self.metrics = []
total_beneficiaries_query = db.session.query(User)
self.metrics.append(metric.Metric(
metric_name='total_beneficiaries',
query=total_beneficiaries_query,
object_model=User,
stock_filters=[filters.beneficiary_filters],
caching_combinatory_strategy=metrics_cache.COUNT,
filterable_by=self.filterable_attributes))
total_vendors_query = db.session.query(User)
self.metrics.append(metric.Metric(
metric_name='total_vendors',
query=total_vendors_query,
object_model=User,
stock_filters=[filters.vendor_filters],
caching_combinatory_strategy=metrics_cache.COUNT,
filterable_by=self.filterable_attributes))
# Timeseries Metrics
if group_strategy:
users_created_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, User.created).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, User.created)), User)
aggregated_users_created_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(User.id).label('volume'), group_strategy.group_by_column), User)
else:
users_created_timeseries_query = db.session.query(func.count(User.id).label('volume'),
func.date_trunc(self.timeseries_unit, User.created).label('date')).group_by(func.date_trunc(self.timeseries_unit, User.created))
aggregated_users_created_query = None
total_users_created_query = db.session.query(func.count(User.id).label('volume'))
self.metrics.append(metric.Metric(
metric_name='users_created',
is_timeseries=True,
query=users_created_timeseries_query,
aggregated_query=aggregated_users_created_query,
total_query=total_users_created_query,
object_model=User,
#stock_filters=[filters.beneficiary_filters], # NOTE: Do we still want this filter?
stock_filters=[],
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
))
if group_strategy:
active_users_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, CreditTransfer.created).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, CreditTransfer.created)), CreditTransfer)
aggregated_active_users_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'), group_strategy.group_by_column), CreditTransfer)
else:
active_users_timeseries_query = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, CreditTransfer.created).label('date')).group_by(func.date_trunc(self.timeseries_unit, CreditTransfer.created))
aggregated_active_users_query = None
total_active_users_query = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'))
self.metrics.append(metric.Metric(
metric_name='active_users',
is_timeseries=True,
query=active_users_timeseries_query,
aggregated_query=aggregated_active_users_query,
total_query=total_active_users_query,
object_model=CreditTransfer,
#stock_filters=[filters.beneficiary_filters], # NOTE: Do we still want this filter?
stock_filters=[],
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
))
# 'default' are
aggregations_by_engine = {
"default": aggregations,
"redshift+psycopg2": aggregations_redshift,
"bigquery": aggregations_bigquery,
}
#######################
# Conversions are a callable on a column expression that yields a
# nonaggregated column expression
# for instance, quarter(sales_date) => func.date_trunc('quarter', MyTable.sales_date)
#######################
conversions = {
"month": lambda fld: func.date_trunc("month", fld),
"week": lambda fld: func.date_trunc("week", fld),
"year": lambda fld: func.date_trunc("year", fld),
"quarter": lambda fld: func.date_trunc("quarter", fld),
"string": lambda fld: func.cast(fld, String()),
"int": lambda fld: func.cast(fld, Integer()),
}
conversions_redshift = {
# age doesn't work on all databases
"age": lambda fld: postgres_age(fld),
}
conversions_bigquery = {
"month": lambda fld: func.date_trunc(fld, text("month")),
def get(self):
"""
Return various stats per-day for the installation.
"""
stat = request.args.get("stat")
if not stat:
return self.error("invalid stat")
if stat not in STAT_CHOICES:
return self.error("invalid stat")
since = request.args.get("since")
if since:
date_end = timezone.fromtimestamp(float(since))
else:
date_end = timezone.now() + timedelta(days=1)
if stat == "users.active":
date_field = User.date_active
elif stat == "users.created":
date_field = User.date_created
else:
date_field = Build.date_created
date_end = date_end.replace(minute=0, second=0, microsecond=0)
resolution = request.args.get("resolution", "1d")
points = int(
request.args.get("points") or stats.POINTS_DEFAULT[resolution])
if resolution == "1h":
grouper = func.date_trunc("hour", date_field)
decr_res = stats.decr_hour
elif resolution == "1d":
grouper = func.date_trunc("day", date_field)
date_end = date_end.replace(hour=0)
decr_res = stats.decr_day
elif resolution == "1w":
grouper = func.date_trunc("week", date_field)
date_end = date_end.replace(hour=0)
date_end -= timedelta(days=date_end.weekday())
decr_res = stats.decr_week
elif resolution == "1m":
grouper = func.date_trunc("month", date_field)
date_end = date_end.replace(hour=0, day=1)
decr_res = stats.decr_month
date_begin = date_end
for _ in range(points):
date_begin = decr_res(date_begin)
if stat.startswith("users."):
queryset = db.session.query(grouper.label("grouper"),
func.count(
User.id)).group_by("grouper")
else:
queryset = stats.build_queryset(stat, grouper)
queryset = queryset.filter(date_field >= date_begin,
date_field < date_end)
queryset = queryset.limit(points)
results = {
# HACK(dcramer): force (but dont convert) the timezone to be utc
# while this isnt correct, we're not looking for correctness yet
k.replace(tzinfo=timezone.utc): v
for k, v in queryset
}
data = []
cur_date = date_end
for _ in range(points):
cur_date = decr_res(cur_date)
data.append({
"time":
int(float(cur_date.strftime("%s.%f")) * 1000),
"value":
(int(float(results[cur_date])) if results.get(cur_date) else
(0 if stat in stats.ZERO_FILLERS else None)),
})
return self.respond(data)
async def get_biggest_day(conn):
day = func.date_trunc('DAY', play.c.date).label('day')
plays = func.count().label('plays')
result = await conn.execute(
select([day, plays]).group_by(day).order_by(plays.desc()).limit(1))
return await result.fetchone()
def get(self, repo: Repository):
"""
Return various stats per-day for the given repository.
"""
stat = request.args.get('stat')
if not stat:
return self.error('missing stat')
if stat not in STAT_CHOICES:
return self.error('invalid stat')
resolution = request.args.get('resolution', '1d')
points = int(request.args.get('points') or POINTS_DEFAULT[resolution])
since = request.args.get('since')
if since:
date_end = datetime.utcfromtimestamp(
float(since)).replace(tzinfo=timezone.utc)
else:
date_end = timezone.now()
date_end = date_end.replace(minute=0, second=0, microsecond=0)
if resolution == '1h':
grouper = func.date_trunc('hour', Build.date_created)
decr_res = decr_hour
elif resolution == '1d':
grouper = func.date_trunc('day', Build.date_created)
date_end = date_end.replace(hour=0)
decr_res = decr_day
elif resolution == '1w':
grouper = func.date_trunc('week', Build.date_created)
date_end = date_end.replace(hour=0)
date_end -= timedelta(days=date_end.weekday())
decr_res = decr_week
elif resolution == '1m':
grouper = func.date_trunc('month', Build.date_created)
date_end = date_end.replace(hour=0, day=1)
decr_res = decr_month
date_begin = date_end
for _ in range(points):
date_begin = decr_res(date_begin)
# TODO(dcramer): put minimum date bounds
if stat in ('builds.aborted', 'builds.failed', 'builds.passed',
'builds.total', 'builds.duration'):
if stat == 'builds.failed':
filters = [Build.result == Result.failed]
elif stat == 'builds.passed':
filters = [Build.result == Result.passed]
elif stat == 'builds.aborted':
filters = [Build.result == Result.aborted]
else:
filters = [Build.status == Status.finished]
if stat == 'builds.duration':
value = func.avg((extract('epoch', Build.date_finished) -
extract('epoch', Build.date_started)) * 1000)
filters = [
Build.status == Status.finished,
Build.result == Result.passed
]
else:
value = func.count(Build.id)
results = {
# HACK(dcramer): force (but dont convert) the timezone to be utc
# while this isnt correct, we're not looking for correctness yet
k.replace(tzinfo=timezone.utc): v
for k, v in db.session.query(
grouper.label('grouper'),
value.label('value'),
).filter(Build.repository_id == repo.id, Build.date_created >=
date_begin, Build.date_created < date_end, *
filters).group_by('grouper')
}
else:
results = {
# HACK(dcramer): force (but dont convert) the timezone to be utc
# while this isnt correct, we're not looking for correctness yet
k.replace(tzinfo=timezone.utc): v
for k, v in db.session.query(
grouper.label('grouper'),
func.avg(ItemStat.value).label('value'),
).filter(
ItemStat.item_id == Build.id,
ItemStat.name == stat,
Build.repository_id == repo.id,
Build.result == Result.passed,
Build.date_created >= date_begin,
Build.date_created < date_end,
).group_by('grouper')
}
data = []
cur_date = date_end
for _ in range(points):
cur_date = decr_res(cur_date)
data.append({
'time': int(float(cur_date.strftime('%s.%f')) * 1000),
'value': int(float(results.get(cur_date) or 0)),
})
# data.reverse()
return self.respond(data)
def calculate_transfer_stats(total_time_series=False,
start_date=None,
end_date=None,
user_filter={}):
date_filter = []
filter_active = False
if start_date is not None and end_date is not None:
date_filter.append(CreditTransfer.created >= start_date)
date_filter.append(CreditTransfer.created <= end_date)
filter_active = True
disbursement_filters = [
CreditTransfer.transfer_status == TransferStatusEnum.COMPLETE,
CreditTransfer.transfer_type == TransferTypeEnum.PAYMENT,
CreditTransfer.transfer_subtype == TransferSubTypeEnum.DISBURSEMENT
]
standard_payment_filters = [
CreditTransfer.transfer_status == TransferStatusEnum.COMPLETE,
CreditTransfer.transfer_type == TransferTypeEnum.PAYMENT,
CreditTransfer.transfer_subtype == TransferSubTypeEnum.STANDARD
]
exchanged_filters = [
CreditTransfer.transfer_status == TransferStatusEnum.COMPLETE,
CreditTransfer.transfer_type == TransferTypeEnum.EXCHANGE,
CreditTransfer.token == g.active_organisation.token
]
beneficiary_filters = [User.has_beneficiary_role == True]
vendor_filters = [User.has_vendor_role == True]
exhaused_balance_filters = [
CreditTransfer.transfer_type == TransferTypeEnum.PAYMENT,
TransferAccount._balance_wei == 0
]
transfer_use_filters = [
*standard_payment_filters,
CreditTransfer.transfer_use.isnot(None),
]
# Disable cache if any filters are being used
disable_cache = False
if user_filter or date_filter:
disable_cache = True
total_distributed = db.session.query(
func.sum(CreditTransfer.transfer_amount).label('total'))
total_distributed = apply_filters(total_distributed, user_filter,
CreditTransfer)
total_distributed = total_distributed.filter(*disbursement_filters).filter(
*date_filter)
total_distributed = metrics_cache.execute_with_partial_history_cache(
'total_distributed',
total_distributed,
CreditTransfer,
metrics_cache.SUM,
disable_cache=disable_cache)
total_spent = db.session.query(
func.sum(CreditTransfer.transfer_amount).label('total'))
total_spent = apply_filters(total_spent, user_filter, CreditTransfer)
total_spent = total_spent.filter(*standard_payment_filters).filter(
*date_filter)
total_spent = metrics_cache.execute_with_partial_history_cache(
'total_spent',
total_spent,
CreditTransfer,
metrics_cache.SUM,
disable_cache=disable_cache)
total_exchanged = db.session.query(
func.sum(CreditTransfer.transfer_amount).label('total'))
total_exchanged = apply_filters(total_exchanged, user_filter,
CreditTransfer)
total_exchanged = total_exchanged.filter(*exchanged_filters).filter(
*date_filter)
total_exchanged = metrics_cache.execute_with_partial_history_cache(
'total_exchanged',
total_exchanged,
CreditTransfer,
metrics_cache.SUM,
disable_cache=disable_cache)
total_beneficiaries = db.session.query(User).filter(*beneficiary_filters)
total_beneficiaries = metrics_cache.execute_with_partial_history_cache(
'total_beneficiaries',
total_beneficiaries,
CreditTransfer,
metrics_cache.COUNT,
disable_cache=disable_cache)
total_vendors = db.session.query(User).filter(*vendor_filters)
total_vendors = metrics_cache.execute_with_partial_history_cache(
'total_vendors',
total_vendors,
CreditTransfer,
metrics_cache.COUNT,
disable_cache=disable_cache)
total_users = total_beneficiaries + total_vendors
has_transferred_count = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id))
.label('transfer_count'))\
.filter(*standard_payment_filters) \
.filter(*date_filter) \
.first().transfer_count
exhausted_balance_count = db.session.query(func.count(func.distinct(
CreditTransfer.sender_transfer_account_id))
.label('transfer_count')) \
.join(CreditTransfer.sender_transfer_account)\
.filter(*exhaused_balance_filters) \
.filter(*date_filter) \
.first().transfer_count
daily_transaction_volume = db.session.query(
func.sum(CreditTransfer.transfer_amount).label('volume'),
func.date_trunc('day', CreditTransfer.created).label('date'))
daily_transaction_volume = apply_filters(daily_transaction_volume,
user_filter, CreditTransfer)
daily_transaction_volume = daily_transaction_volume.group_by(func.date_trunc('day', CreditTransfer.created))\
.filter(*standard_payment_filters) \
.filter(*date_filter)
daily_transaction_volume = metrics_cache.execute_with_partial_history_cache(
'daily_transaction_volume',
daily_transaction_volume,
CreditTransfer,
metrics_cache.SUM_OBJECTS,
disable_cache=disable_cache)
daily_disbursement_volume = db.session.query(
func.sum(CreditTransfer.transfer_amount).label('volume'),
func.date_trunc('day', CreditTransfer.created).label('date'))
daily_disbursement_volume = apply_filters(daily_disbursement_volume,
user_filter, CreditTransfer)
daily_disbursement_volume = daily_disbursement_volume.group_by(func.date_trunc('day', CreditTransfer.created)) \
.filter(*disbursement_filters) \
.filter(*date_filter)
daily_disbursement_volume = metrics_cache.execute_with_partial_history_cache(
'daily_disbursement_volume',
daily_disbursement_volume,
CreditTransfer,
metrics_cache.SUM_OBJECTS,
disable_cache=disable_cache)
transfer_use_breakdown = db.session.query(
CreditTransfer.transfer_use.cast(JSONB),
func.count(CreditTransfer.transfer_use))
transfer_use_breakdown = apply_filters(transfer_use_breakdown, user_filter,
CreditTransfer)
transfer_use_breakdown = transfer_use_breakdown.filter(*transfer_use_filters) \
.group_by(CreditTransfer.transfer_use.cast(JSONB)) \
.all()
try:
last_day = daily_transaction_volume[0][1]
last_day_volume = daily_transaction_volume[0][0]
transaction_vol_list = [{
'date': item[1].isoformat(),
'volume': item[0]
} for item in daily_transaction_volume]
except IndexError: # No transactions
last_day = datetime.datetime.utcnow()
last_day_volume = 0
has_transferred_count = 0
transaction_vol_list = [{
'date': datetime.datetime.utcnow().isoformat(),
'volume': 0
}]
try:
disbursement_vol_list = [{
'date': item[1].isoformat(),
'volume': item[0]
} for item in daily_disbursement_volume]
except IndexError:
disbursement_vol_list = [{
'date': datetime.datetime.utcnow().isoformat(),
'volume': 0
}]
try:
master_wallet_balance = cached_funds_available()
except:
master_wallet_balance = 0
data = {
'total_distributed': total_distributed,
'total_spent': total_spent,
'total_exchanged': total_exchanged,
'has_transferred_count': has_transferred_count,
'zero_balance_count': exhausted_balance_count,
'total_beneficiaries': total_beneficiaries,
'total_users': total_users,
'master_wallet_balance': master_wallet_balance,
'daily_transaction_volume': transaction_vol_list,
'daily_disbursement_volume': disbursement_vol_list,
'transfer_use_breakdown': transfer_use_breakdown,
'last_day_volume': {
'date': last_day.isoformat(),
'volume': last_day_volume
},
'filter_active': filter_active
}
return data
def admin_graph(request):
graph_name = request.matchdict['name']
try:
import pygal
except ImportError:
raise HTTPNotFound()
def get(name, default=None, type=str):
try:
return type(request.GET.get(name, default))
except ValueError:
raise HTTPBadRequest()
def graphround(f):
""" round(), then int() if f is integer """
f = round(f, 4)
if f % 1 == 0:
f = int(f)
return f
pygalopts = {
'js': [
request.static_url('ccvpn:static/pygal/svg.jquery.js'),
request.static_url('ccvpn:static/pygal/pygal-tooltips.js')
]
}
period = get('period', 'm')
if period == 'm':
period_time = timedelta(days=30)
if period == 'y':
period_time = timedelta(days=365)
if graph_name == 'users':
period = get('period', 'm')
chart = pygal.Line(fill=True, x_label_rotation=75, show_legend=False,
**pygalopts)
chart.title = 'Users (%s)' % period
chart.x_labels = []
values = []
gen = last_days(30) if period == 'm' else last_months(12)
users = DBSession.query(User).all()
for m in gen:
filter_ = time_filter_future(period, m, lambda o: o.signup_date)
users_filtered = filter(filter_, users)
values.append(len(list(users_filtered)))
chart.x_labels.append('%s/%s/%s' % (m.year, m.month, m.day))
chart.add('Users', values)
return Response(chart.render(), content_type='image/svg+xml')
elif graph_name == 'income':
currency = get('currency', str)
if currency == 'eur':
graph_methods = (methods.PaypalMethod, methods.StripeMethod)
elif currency == 'btc':
graph_methods = (methods.BitcoinMethod, )
else:
raise HTTPNotFound()
chart = pygal.StackedBar(x_label_rotation=75, show_legend=True,
**pygalopts)
chart.title = 'Income (%s, %s)' % (currency, period)
orders = DBSession.query(Order) \
.filter(Order.start_date > datetime.now() - period_time) \
.filter(Order.paid == True) \
.filter(or_(*(Order.method == m.id for m in graph_methods))) \
.all()
# Prepare value dict
values = {}
for order in orders:
t = order.method
if t not in values:
values[t] = []
chart.x_labels = []
gen = last_days(30) if period == 'm' else last_months(12)
for m in gen:
filter_ = time_filter(period, m, lambda o: o.start_date)
orders_date = list(filter(filter_, orders))
for method in values.keys():
filter_ = lambda o: o.method == method
orders_dd = list(filter(filter_, orders_date))
sum_ = sum(o.paid_amount for o in orders_dd)
values[method].append(graphround(sum_) or None)
chart.x_labels.append('%s' % m)
for method, v in values.items():
label = request.payment_methods[method].name
chart.add(label, v)
return Response(chart.render(), content_type='image/svg+xml')
elif graph_name == 'sessions':
chart = pygal.StackedBar(x_label_rotation=75, show_legend=True,
**pygalopts)
chart.title = 'Sessions (%s)' % (period)
cdate = func.date_trunc('day' if period == 'm' else 'month',
VPNSession.connect_date).label('cdate')
counts = DBSession.query(VPNSession.gateway_id,
VPNSession.gateway_version,
func.count(VPNSession.id).label('count'),
cdate) \
.group_by(VPNSession.gateway_id) \
.group_by(VPNSession.gateway_version) \
.group_by(cdate) \
.all()
values = {} # gw_key / date / count
for item in counts:
gw_key = str(item.gateway_id) + '/' + str(item.gateway_version)
if not gw_key in values:
values[gw_key] = {}
values[gw_key][item.cdate.date()] = item.count
chart.x_labels = []
values2 = {}
gen = last_days(30) if period == 'm' else last_months(12)
for m in gen:
for gw_key, dates in values.items():
if gw_key not in values2:
values2[gw_key] = {}
values2[gw_key][m] = dates.get(m)
chart.x_labels.append('%s' % m)
for gw_key, dates in values2.items():
label = gw_key
sorted_dates = sorted(dates.items())
sorted_counts = map(lambda x: x[1], sorted_dates)
chart.add(label, sorted_counts)
return Response(chart.render(), content_type='image/svg+xml')
else:
raise HTTPNotFound()
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
points = args.points or POINTS_DEFAULT[args.resolution]
if args.from_date:
date_end = datetime.fromtimestamp(args.from_date)
else:
date_end = datetime.now()
date_end = date_end.replace(minute=0, second=0, microsecond=0)
if args.resolution == '1h':
grouper = func.date_trunc('hour', Build.date_created)
decr_res = lambda x: x - timedelta(hours=1)
elif args.resolution == '1d':
grouper = func.date_trunc('day', Build.date_created)
date_end = date_end.replace(hour=0)
decr_res = lambda x: x - timedelta(days=1)
elif args.resolution == '1w':
grouper = func.date_trunc('week', Build.date_created)
date_end = date_end.replace(hour=0)
date_end -= timedelta(days=date_end.weekday())
decr_res = decr_week
elif args.resolution == '1m':
grouper = func.date_trunc('month', Build.date_created)
date_end = date_end.replace(hour=0, day=1)
decr_res = decr_month
if args.agg:
value = getattr(func, args.agg)(ItemStat.value)
else:
value = func.avg(ItemStat.value)
date_begin = date_end.replace()
for _ in xrange(points):
date_begin = decr_res(date_begin)
# TODO(dcramer): put minimum date bounds
results = dict(
db.session.query(
grouper.label('grouper'),
value.label('value'),
).filter(
ItemStat.item_id == Build.id,
ItemStat.name == args.stat,
Build.project_id == project.id,
Build.date_created >= date_begin,
Build.date_created < date_end,
).group_by('grouper'))
data = []
cur_date = date_end.replace()
for _ in xrange(points):
cur_date = decr_res(cur_date)
data.append({
'time': int(float(cur_date.strftime('%s.%f')) * 1000),
'value': int(float(results.get(cur_date, 0))),
})
data.reverse()
return self.respond(data, serialize=False)
def query_work_day_stats(
company_id,
start_date=None,
end_date=None,
first=None,
after=None,
tzname="Europe/Paris",
):
tz = gettz(tzname)
if after:
max_time, user_id_ = parse_datetime_plus_id_cursor(after)
max_date = max_time.date()
end_date = min(max_date, end_date) if end_date else max_date
query = (Activity.query.join(Mission).join(
Expenditure,
and_(
Activity.user_id == Expenditure.user_id,
Activity.mission_id == Expenditure.mission_id,
),
isouter=True,
).with_entities(
Activity.id,
Activity.user_id,
Activity.mission_id,
Mission.name,
Activity.start_time,
Activity.end_time,
Activity.type,
Expenditure.id.label("expenditure_id"),
Expenditure.type.label("expenditure_type"),
func.generate_series(
func.date_trunc(
"day",
func.timezone(
tzname,
func.timezone("UTC", Activity.start_time),
),
),
func.timezone(
tzname,
func.coalesce(
func.timezone("UTC", Activity.end_time),
func.now(),
),
),
"1 day",
).label("day"),
).filter(
Mission.company_id == company_id,
~Activity.is_dismissed,
Activity.start_time != Activity.end_time,
))
query = _apply_time_range_filters(
query,
to_datetime(start_date, tz_for_date=tz),
to_datetime(end_date,
tz_for_date=tz,
convert_dates_to_end_of_day_times=True),
)
has_next_page = False
if first:
activity_first = max(first * 5, 200)
query = query.order_by(desc("day"), desc(
Activity.user_id)).limit(activity_first + 1)
has_next_page = query.count() > activity_first
query = query.subquery()
query = (db.session.query(query).group_by(
query.c.user_id, query.c.day, query.c.mission_id,
query.c.name).with_entities(
query.c.user_id.label("user_id"),
query.c.day,
func.timezone("UTC",
func.timezone(tzname,
query.c.day)).label("utc_day_start"),
query.c.mission_id.label("mission_id"),
query.c.name.label("mission_name"),
func.min(
func.greatest(
query.c.start_time,
func.timezone("UTC", func.timezone(tzname, query.c.day)),
)).label("start_time"),
func.max(
func.least(
func.timezone(
"UTC",
func.timezone(
tzname,
query.c.day + func.cast("1 day", Interval)),
),
func.coalesce(query.c.end_time, func.now()),
)).label("end_time"),
func.bool_or(
and_(
query.c.end_time.is_(None),
query.c.day == func.current_date(),
)).label("is_running"),
*[
func.sum(
case(
[(
query.c.type == a_type.value,
extract(
"epoch",
func.least(
func.timezone(
"UTC",
func.timezone(
tzname,
query.c.day +
func.cast("1 day", Interval),
),
),
func.coalesce(query.c.end_time,
func.now()),
) - func.greatest(
query.c.start_time,
func.timezone(
"UTC",
func.timezone(tzname, query.c.day),
),
),
),
)],
else_=0,
)).label(f"{a_type.value}_duration")
for a_type in ActivityType
],
func.greatest(func.count(distinct(query.c.expenditure_id)),
1).label("n_exp_dups"),
func.count(distinct(query.c.id)).label("n_act_dups"),
*[
func.sum(
case(
[(query.c.expenditure_type == e_type.value, 1)],
else_=0,
)).label(f"n_{e_type.value}_expenditures")
for e_type in ExpenditureType
],
).subquery())
query = (db.session.query(query).group_by(
query.c.user_id, query.c.day).with_entities(
query.c.user_id.label("user_id"),
query.c.day,
func.array_agg(distinct(
query.c.mission_name)).label("mission_names"),
func.min(query.c.start_time).label("start_time"),
func.max(query.c.end_time).label("end_time"),
func.bool_or(query.c.is_running).label("is_running"),
*[
func.sum(
getattr(query.c, f"{a_type.value}_duration") /
query.c.n_exp_dups).cast(Integer).label(
f"{a_type.value}_duration") for a_type in ActivityType
],
*[
func.sum(
getattr(query.c, f"n_{e_type.value}_expenditures") /
query.c.n_act_dups).cast(Integer).label(
f"n_{e_type.value}_expenditures")
for e_type in ExpenditureType
],
).order_by(desc("day"), desc("user_id")).subquery())
query = db.session.query(query).with_entities(
*query.c,
extract("epoch", query.c.end_time -
query.c.start_time).label("service_duration"),
reduce(
lambda a, b: a + b,
[
getattr(query.c, f"{a_type.value}_duration")
for a_type in ActivityType
],
).label("total_work_duration"),
)
results = query.all()
if after:
results = [
r for r in results if r.day.date() < max_date or (
r.day.date() == max_date and r.user_id < user_id_)
]
if first:
if has_next_page:
# The last work day may be incomplete because we didn't fetch all the activities => remove it
results = results[:-1]
if len(results) > first:
results = results[:first]
has_next_page = True
return results, has_next_page
def setUp(self):
engine = create_engine('postgresql://postgres@localhost/pypet')
self.metadata = MetaData(bind=engine)
self.store_table = Table('store', self.metadata,
Column('store_id', types.Integer, primary_key=True),
Column('store_name', types.String),
Column('country_id', types.Integer,
ForeignKey('country.country_id')))
self.country_table = Table('country', self.metadata,
Column('country_id', types.Integer, primary_key=True),
Column('country_name', types.String),
Column('region_id', types.Integer,
ForeignKey('region.region_id')))
self.region_table = Table('region', self.metadata,
Column('region_id', types.Integer, primary_key=True),
Column('region_name', types.String))
self.product_table = Table('product', self.metadata,
Column('product_id', types.Integer, primary_key=True),
Column('product_name', types.String),
Column('product_category_id', types.Integer,
ForeignKey('product_category.product_category_id')))
self.product_category_table = Table('product_category', self.metadata,
Column('product_category_id', types.Integer, primary_key=True),
Column('product_category_name', types.String))
self.facts_table = Table('facts_table', self.metadata,
Column('store_id', types.Integer,
ForeignKey('store.store_id')),
Column('date', types.Date),
Column('product_id', types.Integer,
ForeignKey('product.product_id')),
Column('price', types.Float),
Column('qty', types.Integer))
agg_name = ('agg_time_month_product_product_store_store'
'_Unit Price_Quantity')
self.agg_by_month_table = Table(agg_name,
self.metadata,
Column('store_store', types.Integer,
ForeignKey('store.store_id')),
Column('time_month', types.Date),
Column('product_product', types.Integer,
ForeignKey('product.product_id')),
Column('Unit Price', types.Float),
Column('Quantity', types.Integer),
Column('fact_count', types.Integer))
agg_name = ('agg_time_year_store_country_product_product'
'_Unit Price_Quantity')
self.agg_by_year_country_table = Table(agg_name,
self.metadata,
Column('store_country', types.Integer,
ForeignKey('country.country_id')),
Column('time_year', types.Date),
Column('product_product', types.Integer,
ForeignKey('product.product_id')),
Column('Unit Price', types.Float),
Column('Quantity', types.Integer),
Column('fact_count', types.Integer))
self.metadata.create_all()
self.store_dim = Dimension('store', [
Hierarchy('default', [
Level('region', self.region_table.c.region_id,
self.region_table.c.region_name),
Level('country', self.country_table.c.country_id,
self.country_table.c.country_name),
Level('store', self.store_table.c.store_id,
self.store_table.c.store_name)])])
self.product_dim = Dimension('product', [
Hierarchy('default', [
Level('category',
self.product_category_table.c.product_category_id,
self.product_category_table.c
.product_category_name),
Level('product', self.product_table.c.product_id,
self.product_table.c.product_name)])])
self.time_dim = TimeDimension('time', self.facts_table.c.date,
['year', 'month', 'day'])
unit_price = Measure('Unit Price', self.facts_table.c.price,
aggregates.avg)
quantity = Measure('Quantity', self.facts_table.c.qty, aggregates.sum)
price = ((unit_price.aggregate_with(None) *
quantity.aggregate_with(None))
.aggregate_with(aggregates.sum).label('Price'))
self.cube = Cube(self.metadata, self.facts_table, [self.store_dim,
self.product_dim, self.time_dim], [unit_price, quantity, price],
fact_count_column=self.facts_table.c.qty)
self.region_table.insert({'region_id': 1, 'region_name':
'Europe'}).execute()
self.country_table.insert({'region_id': 1, 'country_name':
'France', 'country_id': 1}).execute()
self.country_table.insert({'region_id': 1, 'country_name':
'Germany', 'country_id': 2}).execute()
self.region_table.insert({'region_id': 2, 'region_name':
'America'}).execute()
self.country_table.insert({'region_id': 2, 'country_name':
'USA', 'country_id': 3}).execute()
self.country_table.insert({'region_id': 2, 'country_name':
'Canada', 'country_id': 4}).execute()
self.store_table.insert({
'store_id': 1,
'store_name': 'ACME.fr',
'country_id': 1}).execute()
self.store_table.insert({
'store_id': 2,
'store_name': 'ACME.de',
'country_id': 2}).execute()
self.store_table.insert({
'store_id': 3,
'store_name': 'Food Mart.fr',
'country_id': 1}).execute()
self.store_table.insert({
'store_id': 4,
'store_name': 'Food Mart.de',
'country_id': 2}).execute()
self.store_table.insert({
'store_id': 5,
'store_name': 'ACME.us',
'country_id': 3}).execute()
self.store_table.insert({
'store_id': 6,
'store_name': 'Food Mart.us',
'country_id': 3}).execute()
self.store_table.insert({
'store_id': 7,
'store_name': 'ACME.ca',
'country_id': 4}).execute()
self.store_table.insert({
'store_id': 8,
'store_name': 'Food Mart.ca',
'country_id': 4}).execute()
self.product_category_table.insert({
'product_category_id': 1,
'product_category_name': 'Vegetables'}).execute()
self.product_category_table.insert({
'product_category_id': 2,
'product_category_name': 'Shoes'}).execute()
self.product_table.insert({
'product_id': 1,
'product_category_id': 1,
'product_name': 'Carrots'}).execute()
self.product_table.insert({
'product_id': 2,
'product_category_id': 1,
'product_name': 'Bananas'}).execute()
self.product_table.insert({
'product_id': 3,
'product_category_id': 2,
'product_name': 'Red shoes'}).execute()
self.product_table.insert({
'product_id': 4,
'product_category_id': 2,
'product_name': 'Green shoes'}).execute()
self.product_table.insert({
'product_id': 5,
'product_category_id': 2,
'product_name': 'Blue shoes'}).execute()
years = cycle([2009, 2010, 2011])
months = cycle([1, 5, 8, 9, 11])
days = cycle([3, 12, 21, 29])
prices = iter(cycle([100, 500, 1000]))
quantities = iter(cycle([1, 5, 1, 2, 3, 20, 8]))
values = iter((date(*value) for value in izip(years, months, days)))
for value in self.product_table.select().with_only_columns([
self.product_table.c.product_id,
self.store_table.c.store_id]).execute():
self.facts_table.insert({
'product_id': value.product_id,
'store_id': value.store_id,
'date': next(values),
'qty': next(quantities),
'price': next(prices)}).execute()
results = (self.facts_table.select().with_only_columns([
(func.sum(self.facts_table.c.price *
self.facts_table.c.qty) /
func.sum(self.facts_table.c.qty))
.label('Unit Price'),
func.sum(self.facts_table.c.qty).label('Quantity'),
func.sum(self.facts_table.c.qty).label('fact_count'),
self.facts_table.c.product_id.label('product_product'),
self.facts_table.c.store_id.label('store_store'),
func.date_trunc('month',
self.facts_table.c.date).label('time_month')])
.group_by(func.date_trunc('month', self.facts_table.c.date),
self.facts_table.c.product_id,
self.facts_table.c.store_id)
.execute())
for res in results:
self.agg_by_month_table.insert().execute(dict(res))
second_agg = (self.facts_table.select().with_only_columns([
(func.sum(self.facts_table.c.price *
self.facts_table.c.qty) /
func.sum(self.facts_table.c.qty))
.label('Unit Price'),
func.sum(self.facts_table.c.qty).label('Quantity'),
func.sum(self.facts_table.c.qty).label('fact_count'),
self.facts_table.c.product_id.label('product_product'),
self.store_table.c.country_id.label('store_country'),
func.date_trunc('year',
self.facts_table.c.date).label('time_year')])
.where(self.facts_table.c.store_id == self.store_table.c.store_id)
.group_by(self.facts_table.c.product_id.label('product_product'),
self.store_table.c.country_id.label('store_country'),
func.date_trunc('year',
self.facts_table.c.date).label('time_year'))
.execute())
for res in second_agg:
self.agg_by_year_country_table.insert().execute(dict(res))
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
points = args.points or POINTS_DEFAULT[args.resolution]
if args.from_date:
date_end = datetime.fromtimestamp(args.from_date)
else:
date_end = datetime.now()
date_end = date_end.replace(
minute=0, second=0, microsecond=0)
if args.resolution == '1h':
grouper = func.date_trunc('hour', Build.date_created)
decr_res = lambda x: x - timedelta(hours=1)
elif args.resolution == '1d':
grouper = func.date_trunc('day', Build.date_created)
date_end = date_end.replace(hour=0)
decr_res = lambda x: x - timedelta(days=1)
elif args.resolution == '1w':
grouper = func.date_trunc('week', Build.date_created)
date_end = date_end.replace(hour=0)
date_end -= timedelta(days=date_end.weekday())
decr_res = decr_week
elif args.resolution == '1m':
grouper = func.date_trunc('month', Build.date_created)
date_end = date_end.replace(hour=0, day=1)
decr_res = decr_month
if args.agg:
value = getattr(func, args.agg)(ItemStat.value)
else:
value = func.avg(ItemStat.value)
date_begin = date_end.replace()
for _ in xrange(points):
date_begin = decr_res(date_begin)
# TODO(dcramer): put minimum date bounds
results = dict(db.session.query(
grouper.label('grouper'),
value.label('value'),
).filter(
ItemStat.item_id == Build.id,
ItemStat.name == args.stat,
Build.project_id == project.id,
Build.date_created >= date_begin,
Build.date_created < date_end,
).group_by('grouper'))
data = []
cur_date = date_end.replace()
for _ in xrange(points):
cur_date = decr_res(cur_date)
data.append({
'time': int(float(cur_date.strftime('%s.%f')) * 1000),
'value': int(float(results.get(cur_date, 0))),
})
data.reverse()
return self.respond(data, serialize=False)
def calculate_transfer_stats(total_time_series=False):
total_distributed = db.session.query(func.sum(models.CreditTransfer.transfer_amount).label('total'))\
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.DISBURSEMENT).first().total
total_spent = db.session.query(func.sum(models.CreditTransfer.transfer_amount).label('total'))\
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.PAYMENT).first().total
total_beneficiaries = db.session.query(
models.User).filter(models.User.is_beneficiary == True).count()
total_vendors = db.session.query(models.User)\
.filter(models.User.is_vendor == True).count()
total_users = total_beneficiaries + total_vendors
has_transferred_count = db.session.query(func.count(func.distinct(models.CreditTransfer.sender_user_id))
.label('transfer_count'))\
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.PAYMENT).first().transfer_count
# zero_balance_count = db.session.query(func.count(models.TransferAccount.id).label('zero_balance_count'))\
# .filter(models.TransferAccount.balance == 0).first().zero_balance_count
exhausted_balance_count = db.session.query(func.count(func.distinct(models.CreditTransfer.sender_transfer_account_id))
.label('transfer_count')) \
.join(models.CreditTransfer.sender_transfer_account)\
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.PAYMENT) \
.filter(models.TransferAccount.balance == 0).first().transfer_count
daily_transaction_volume = db.session.query(func.sum(models.CreditTransfer.transfer_amount).label('volume'),
func.date_trunc('day', models.CreditTransfer.created).label('date'))\
.group_by(func.date_trunc('day', models.CreditTransfer.created))\
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.PAYMENT).all()
daily_disbursement_volume = db.session.query(func.sum(models.CreditTransfer.transfer_amount).label('volume'),
func.date_trunc('day', models.CreditTransfer.created).label('date')) \
.group_by(func.date_trunc('day', models.CreditTransfer.created)) \
.filter(models.CreditTransfer.transfer_type == models.TransferTypeEnum.DISBURSEMENT).all()
try:
master_wallet_balance = master_wallet_funds_available()
except BlockchainError:
master_wallet_balance = 0
try:
last_day = daily_transaction_volume[0].date
last_day_volume = daily_transaction_volume[0].volume
transaction_vol_list = [{
'date': item.date.isoformat(),
'volume': item.volume
} for item in daily_transaction_volume]
except IndexError: # No transactions
last_day = datetime.datetime.utcnow()
last_day_volume = 0
has_transferred_count = 0
transaction_vol_list = [{
'date': datetime.datetime.utcnow().isoformat(),
'volume': 0
}]
try:
last_day_disbursement_volume = daily_disbursement_volume[0].volume
disbursement_vol_list = [{
'date': item.date.isoformat(),
'volume': item.volume
} for item in daily_disbursement_volume]
except IndexError:
last_day_disbursement_volume = 0
disbursement_vol_list = [{
'date': datetime.datetime.utcnow().isoformat(),
'volume': 0
}]
data = {
'total_distributed': total_distributed,
'total_spent': total_spent,
'has_transferred_count': has_transferred_count,
'zero_balance_count': exhausted_balance_count,
'total_beneficiaries': total_beneficiaries,
'total_users': total_users,
'master_wallet_balance': master_wallet_balance,
'daily_transaction_volume': transaction_vol_list,
'daily_disbursement_volume': disbursement_vol_list,
'last_day_volume': {
'date': last_day.isoformat(),
'volume': last_day_volume
}
}
return data
def date_of_posting(cls):
return cast(func.date_trunc('day', cls.posted), DATE)
def groupby_created(precision='month'):
conn = sa.connect()
month = func.date_trunc('month', users.c.created).label('month')
q = select([month, func.count(users.c.id)]).group_by('month').order_by('month')
#return conn.execute(q).fetchall()
return [(dt.strftime('%b %Y'), num) for (dt, num) in conn.execute(q).fetchall()]
def __init__(self, group_strategy, timeseries_unit = 'day', token=None, date_filter_attributes=None):
self.filterable_attributes = [DATE, CUSTOM_ATTRIBUTE, TRANSFER_ACCOUNT, CREDIT_TRANSFER, USER]
self.timeseries_unit = timeseries_unit
self.date_filter_attributes = date_filter_attributes
self.metrics = []
total_amount_query = db.session.query(func.sum(CreditTransfer.transfer_amount).label('total'))
self.metrics.append(metric.Metric(
metric_name='total_distributed',
query=total_amount_query,
object_model=CreditTransfer,
stock_filters=[filters.disbursement_filters],
caching_combinatory_strategy=metrics_cache.SUM,
filterable_by=self.filterable_attributes,
bypass_user_filters=True,
))
self.metrics.append(metric.Metric(
metric_name='total_reclaimed',
query=total_amount_query,
object_model=CreditTransfer,
stock_filters=[filters.reclamation_filters],
caching_combinatory_strategy=metrics_cache.SUM,
filterable_by=self.filterable_attributes,
bypass_user_filters=True,
))
self.metrics.append(metric.Metric(
metric_name='total_withdrawn',
query=total_amount_query,
object_model=CreditTransfer,
stock_filters=[filters.withdrawal_filters],
caching_combinatory_strategy=metrics_cache.SUM,
filterable_by=self.filterable_attributes,
bypass_user_filters=True,
))
# Timeseries Metrics
if group_strategy:
transaction_volume_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.sum(CreditTransfer.transfer_amount).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer])), CreditTransfer)
aggregated_transaction_volume_query = group_strategy.build_query_group_by_with_join(db.session.query(func.sum(CreditTransfer.transfer_amount).label('volume'), group_strategy.group_by_column), CreditTransfer)
else:
transaction_volume_timeseries_query = db.session.query(func.sum(CreditTransfer.transfer_amount).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date')).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]))
aggregated_transaction_volume_query = None
total_transaction_volume_query = db.session.query(func.sum(CreditTransfer.transfer_amount).label('volume'))
self.metrics.append(metric.Metric(
metric_name='all_payments_volume',
is_timeseries=True,
query=transaction_volume_timeseries_query,
aggregated_query=aggregated_transaction_volume_query,
total_query=total_transaction_volume_query,
object_model=CreditTransfer,
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
value_type=CURRENCY,
token=token
))
self.metrics.append(metric.Metric(
metric_name='transfer_amount_per_user',
is_timeseries=True,
query=transaction_volume_timeseries_query,
aggregated_query=aggregated_transaction_volume_query,
total_query=total_transaction_volume_query,
object_model=CreditTransfer,
stock_filters=[filters.standard_payment_filters],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[CALCULATE_TIMESERIES_PER_USER, FORMAT_TIMESERIES], # Add per user
aggregated_query_actions=[CALCULATE_AGGREGATE_PER_USER, FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST, CALCULATE_TOTAL_PER_USER],
value_type=CURRENCY,
token=token
))
if group_strategy:
transaction_count_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(CreditTransfer.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer])), CreditTransfer)
aggregated_transaction_count_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(CreditTransfer.id).label('volume'), group_strategy.group_by_column), CreditTransfer)
else:
transaction_count_timeseries_query = db.session.query(func.count(CreditTransfer.id).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date')).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]))
aggregated_transaction_count_query = None
total_transaction_count_query = db.session.query(func.count(CreditTransfer.id).label('volume'))
self.metrics.append(metric.Metric(
metric_name='daily_transaction_count',
is_timeseries=True,
query=transaction_count_timeseries_query,
aggregated_query=aggregated_transaction_count_query,
total_query=total_transaction_count_query,
object_model=CreditTransfer,
stock_filters=[filters.standard_payment_filters],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
value_type=COUNT
))
self.metrics.append(metric.Metric(
metric_name='trades_per_user',
is_timeseries=True,
query=transaction_count_timeseries_query,
aggregated_query=aggregated_transaction_count_query,
total_query=total_transaction_count_query,
object_model=CreditTransfer,
stock_filters=[filters.standard_payment_filters],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[CALCULATE_TIMESERIES_PER_USER, FORMAT_TIMESERIES], # Add per user
aggregated_query_actions=[CALCULATE_AGGREGATE_PER_USER, FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST, CALCULATE_TOTAL_PER_USER],
value_type=COUNT_AVERAGE,
))
if group_strategy:
active_users_timeseries_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date'), group_strategy.group_by_column).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer])), CreditTransfer)
aggregated_active_users_query = group_strategy.build_query_group_by_with_join(db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'), group_strategy.group_by_column), CreditTransfer)
else:
active_users_timeseries_query = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'),
func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]).label('date')).group_by(func.date_trunc(self.timeseries_unit, self.date_filter_attributes[CreditTransfer]))
aggregated_active_users_query = None
total_transaction_volume_query = db.session.query(func.count(func.distinct(CreditTransfer.sender_user_id)).label('volume'))
self.metrics.append(metric.Metric(
metric_name='users_who_made_purchase',
is_timeseries=True,
query=active_users_timeseries_query,
aggregated_query=aggregated_active_users_query,
total_query=total_transaction_volume_query,
object_model=CreditTransfer,
#stock_filters=[filters.beneficiary_filters], # NOTE: Do we want this filter?
stock_filters=[],
timeseries_caching_combinatory_strategy=metrics_cache.SUM_OBJECTS,
caching_combinatory_strategy=metrics_cache.QUERY_ALL,
filterable_by=self.filterable_attributes,
query_actions=[FORMAT_TIMESERIES],
aggregated_query_actions=[FORMAT_AGGREGATE_METRICS],
total_query_actions=[GET_FIRST],
value_type=COUNT,
))
def partial_trunc(column):
return func.date_trunc(time_slice, column)
def admin_graph(request):
graph_name = request.matchdict['name']
try:
import pygal
except ImportError:
raise HTTPNotFound()
def get(name, default=None, type=str):
try:
return type(request.GET.get(name, default))
except ValueError:
raise HTTPBadRequest()
def graphround(f):
""" round(), then int() if f is integer """
f = round(f, 4)
if f % 1 == 0:
f = int(f)
return f
pygalopts = {
'js': [
request.static_url('ccvpn:static/pygal/svg.jquery.js'),
request.static_url('ccvpn:static/pygal/pygal-tooltips.js')
]
}
period = get('period', 'm')
if period == 'm':
period_time = timedelta(days=30)
if period == 'y':
period_time = timedelta(days=365)
if graph_name == 'users':
period = get('period', 'm')
chart = pygal.Line(fill=True,
x_label_rotation=75,
show_legend=False,
**pygalopts)
chart.title = 'Users (%s)' % period
chart.x_labels = []
values = []
gen = last_days(30) if period == 'm' else last_months(12)
users = DBSession.query(User).all()
for m in gen:
filter_ = time_filter_future(period, m, lambda o: o.signup_date)
users_filtered = filter(filter_, users)
values.append(len(list(users_filtered)))
chart.x_labels.append('%s/%s/%s' % (m.year, m.month, m.day))
chart.add('Users', values)
return Response(chart.render(), content_type='image/svg+xml')
elif graph_name == 'income':
currency = get('currency', str)
if currency == 'eur':
graph_methods = (methods.PaypalMethod, methods.StripeMethod)
elif currency == 'btc':
graph_methods = (methods.BitcoinMethod, )
else:
raise HTTPNotFound()
chart = pygal.StackedBar(x_label_rotation=75,
show_legend=True,
**pygalopts)
chart.title = 'Income (%s, %s)' % (currency, period)
orders = DBSession.query(Order) \
.filter(Order.start_date > datetime.now() - period_time) \
.filter(Order.paid == True) \
.filter(or_(*(Order.method == m.id for m in graph_methods))) \
.all()
# Prepare value dict
values = {}
for order in orders:
t = order.method
if t not in values:
values[t] = []
chart.x_labels = []
gen = last_days(30) if period == 'm' else last_months(12)
for m in gen:
filter_ = time_filter(period, m, lambda o: o.start_date)
orders_date = list(filter(filter_, orders))
for method in values.keys():
filter_ = lambda o: o.method == method
orders_dd = list(filter(filter_, orders_date))
sum_ = sum(o.paid_amount for o in orders_dd)
values[method].append(graphround(sum_) or None)
chart.x_labels.append('%s' % m)
for method, v in values.items():
label = request.payment_methods[method].name
chart.add(label, v)
return Response(chart.render(), content_type='image/svg+xml')
elif graph_name == 'sessions':
chart = pygal.StackedBar(x_label_rotation=75,
show_legend=True,
**pygalopts)
chart.title = 'Sessions (%s)' % (period)
cdate = func.date_trunc('day' if period == 'm' else 'month',
VPNSession.connect_date).label('cdate')
counts = DBSession.query(VPNSession.gateway_id,
VPNSession.gateway_version,
func.count(VPNSession.id).label('count'),
cdate) \
.group_by(VPNSession.gateway_id) \
.group_by(VPNSession.gateway_version) \
.group_by(cdate) \
.all()
values = {} # gw_key / date / count
for item in counts:
gw_key = str(item.gateway_id) + '/' + str(item.gateway_version)
if gw_key not in values:
values[gw_key] = {}
values[gw_key][item.cdate.date()] = item.count
chart.x_labels = []
values2 = {}
gen = last_days(30) if period == 'm' else last_months(12)
for m in gen:
for gw_key, dates in values.items():
if gw_key not in values2:
values2[gw_key] = {}
values2[gw_key][m] = dates.get(m)
chart.x_labels.append('%s' % m)
for gw_key, dates in values2.items():
label = gw_key
sorted_dates = sorted(dates.items())
sorted_counts = map(lambda x: x[1], sorted_dates)
chart.add(label, sorted_counts)
return Response(chart.render(), content_type='image/svg+xml')
else:
raise HTTPNotFound()