def _get_running_totals(query, cm_fields, fid_cache, cont_values, total_data):
# Filter out already added columns, and then add remaining, if they are metrics, replace null with 0
# Skip filtering first column (function_id), since this is a special case and not used when dumping data
for field in [field for field in cm_fields if field not in [c['name'] for c in query.column_descriptions[1:]]]:
if field in MetricsDb.METRICS:
col = replace_null_0(ChangeMetric().columnitems[field])
else:
col = ChangeMetric().columnitems[field]
query = query.add_columns(col)
column_offset = len(query.column_descriptions) - len(cm_fields)
for cm in query:
field_values = []
continous_index = 0
'''
cm[0] == function_id
cm[1:] == other fields in order they where supplied in the request
But the continous values do have special consideration that we need to get previous values. To identify
if its a continous field we check if its part of the cont_values dict, if so, we know its a cont. field.
Then we have a special cont_index which keeps track and which cont. value to append in the list.
'''
prev_values = fid_cache.get(cm[0], array.array('i', (0,) * len(cont_values)))
for index, field in enumerate(cm_fields):
# total_values only contain continous values!
if field in cont_values:
cont_values[field] += cm[index + column_offset] - prev_values[continous_index]
prev_values[continous_index] = cm[index + column_offset]
field_values.append(cont_values[field])
continous_index += 1
else:
field_values.append(cm[index + 1])
fid_cache[cm[0]] = prev_values
total_data.append(field_values)
return total_data
def _get_initial_continous_data(session, params):
continous_fields = [field for field in params['fields'] if field in MetricsDb.METRICS
and MetricsDb.METRICS[field]['continuous']]
total_values = {field: 0 for field in continous_fields}
fid_cache = {}
if len(continous_fields) > 0:
# Get initial values for all functions
# We need to do this to get a baseline metric if we are filtering with "from=" clause.
sq = session.query(ChangeMetric.function_id, func.max(ChangeMetric.date).label('date'))\
.filter(ChangeMetric.date < params['from'])\
.group_by(ChangeMetric.function_id)\
.subquery('prev_nloc')
query = session.query(ChangeMetric.function_id).filter(and_(ChangeMetric.function_id == sq.c.function_id,
ChangeMetric.date == sq.c.date))
for field in continous_fields:
query = query.add_columns(replace_null_0(ChangeMetric().columnitems[field]))
for cm in query:
fid_cache[cm[0]] = array.array('i', cm[1:])
for index, field in enumerate(continous_fields):
total_values[field] += cm[index + 1]
return fid_cache, total_values
def _get_change_metric_snapshot(session, params, query):
'''
Builds 2 queries, one for continous data and one for discontinous data.
Any non-metric fields gets added to the discontinous query.
Then we iterate over each query and update an intermediate dictionary, essentially joining both queries into
a dict with first column as key.
Then we create a new list where we iterate over the dictionary, but adding fields to the new list in the order
they where supplied in the fields parameter.
This is likely very inefficient compared to if we could join both queries, and then directly return the result
back for transformation. Unfortunatly I have been unable to join queries accuratly due to that continous data
need to query data over values with date<fields['from'], while discontinous data needs to filter between dates.
This might be possible using CTE, but I have not yet been able to get that to work accuratly.
'''
sq = session.query(ChangeMetric.function_id, func.max(ChangeMetric.date).label('date'))\
.filter(ChangeMetric.date <= params['to'])\
.group_by(ChangeMetric.function_id)\
.subquery('max_func')
cont_query = query.filter(and_(ChangeMetric.function_id == sq.c.function_id,
ChangeMetric.date == sq.c.date))
disc_query = query.filter(ChangeMetric.date.between(params['from'], params['to']))
for field in params['fields']:
if field in MetricsDb.METRICS:
colclause = func.sum(replace_null_0(ChangeMetric().columnitems[field])).label(field)
if MetricsDb.METRICS[field]['continuous']:
cont_query = cont_query.add_columns(colclause)
else:
disc_query = disc_query.add_columns(colclause)
elif field in ChangeMetric().columnitems:
colclause = func.max(ChangeMetric().columnitems[field]).label(field)
disc_query = disc_query.add_columns(colclause)
data_dict = {}
for c, d in zip(cont_query, disc_query):
data_dict.setdefault(c[0], {}).update(c._asdict())
data_dict.setdefault(d[0], {}).update(d._asdict())
data = []
for r in data_dict.itervalues():
row = []
for field in params['fields']:
assert field in r
row.append(r[field])
data.append(row)
return data