def augment_solr_response(response, collection, query):
augmented = response
augmented['normalized_facets'] = []
NAME = '%(field)s-%(id)s'
normalized_facets = []
selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
if response and response.get('facet_counts'):
for facet in collection['facets']:
category = facet['type']
if category == 'field' and response['facet_counts']['facet_fields']:
name = NAME % facet
collection_facet = get_facet_field(category, name, collection['facets'])
counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name])
if collection_facet['properties']['sort'] == 'asc':
counts.reverse()
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
}
normalized_facets.append(facet)
elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']:
name = NAME % facet
collection_facet = get_facet_field(category, name, collection['facets'])
counts = response['facet_counts']['facet_ranges'][name]['counts']
end = response['facet_counts']['facet_ranges'][name]['end']
counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
'extraSeries': []
}
normalized_facets.append(facet)
elif category == 'query' and response['facet_counts']['facet_queries']:
for name, value in response['facet_counts']['facet_queries'].iteritems():
collection_facet = get_facet_field(category, name, collection['facets'])
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': value,
}
normalized_facets.append(facet)
elif category == 'pivot':
name = NAME % facet
if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']:
if facet['properties']['scope'] == 'stack':
count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values)
else:
count = response['facet_counts']['facet_pivot'][name]
_augment_pivot_nd(facet['id'], count, selected_values)
else:
count = []
facet = {
'id': facet['id'],
'field': name,
'type': category,
'label': name,
'counts': count,
}
normalized_facets.append(facet)
if response and response.get('facets'):
for facet in collection['facets']:
category = facet['type']
name = facet['id'] # Nested facets can only have one name
if category == 'function' and name in response['facets']:
value = response['facets'][name]
collection_facet = get_facet_field(category, name, collection['facets'])
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': value,
}
normalized_facets.append(facet)
elif category == 'nested' and name in response['facets']:
value = response['facets'][name]
collection_facet = get_facet_field(category, name, collection['facets'])
extraSeries = []
counts = response['facets'][name]['buckets']
cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
last_x_col = 0
last_xx_col = 0
for i, f in enumerate(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
cols.append(f['field'])
last_xx_col = last_x_col
last_x_col = i + 2
cols.append(SolrApi._get_aggregate_function(f))
rows = []
# For dim in dimensions
# Number or Date range
if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
dimension = 3
# Single dimension or dimension 2 with analytics
if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1:
column = 'count'
if len(collection_facet['properties']['facets']) == 1:
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
legend = agg_keys[0].split(':', 2)[1]
column = agg_keys[0]
else:
legend = facet['field'] # 'count(%s)' % legend
agg_keys = [column]
_augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
else:
# Dimension 1 with counts and 2 with analytics
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
_series = collections.defaultdict(list)
for row in rows:
for i, cell in enumerate(row):
if i > last_x_col:
legend = cols[i]
if last_xx_col != last_x_col:
legend = '%s %s' % (cols[i], row[last_x_col])
_series[legend].append(row[last_xx_col])
_series[legend].append(cell)
for name, val in _series.iteritems():
_c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet)
extraSeries.append({'counts': _c, 'label': name})
counts = []
elif collection_facet['properties'].get('isOldPivot'):
facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count']
column = 'count'
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
# _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
print counts
# count = response['facets'][name]
# _convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
dimension = len(facet_fields)
elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
# Dimension 1 with 1 count or agg
dimension = 1
column = 'count'
if len(collection_facet['properties']['facets']) == 1:
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
legend = agg_keys[0].split(':', 2)[1]
column = agg_keys[0]
else:
legend = facet['field']
agg_keys = [column]
_augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
counts = pairwise2(legend, selected_values.get(facet['id'], []), counts)
else:
# Dimension 2 with analytics or 1 with N aggregates
dimension = 2
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
'extraSeries': extraSeries,
'dimension': dimension,
'response': {'response': {'start': 0, 'numFound': response['facets'][name]['numBuckets']}}, # Todo * nested buckets + offsets
'docs': [dict(zip(cols, row)) for row in rows],
'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols]
}
normalized_facets.append(facet)
# Remove unnecessary facet data
if response:
response.pop('facet_counts')
response.pop('facets')
# HTML escaping
if not query.get('download'):
id_field = collection.get('idField', '')
for doc in response['response']['docs']:
for field, value in doc.iteritems():
if isinstance(value, numbers.Number):
escaped_value = value
elif field == '_childDocuments_': # Nested documents
escaped_value = value
elif isinstance(value, list): # Multivalue field
escaped_value = [smart_unicode(escape(val), errors='replace') for val in value]
else:
value = smart_unicode(value, errors='replace')
escaped_value = escape(value)
doc[field] = escaped_value
link = None
if 'link-meta' in doc:
meta = json.loads(doc['link-meta'])
link = get_data_link(meta)
doc['externalLink'] = link
doc['details'] = []
doc['hueId'] = smart_unicode(doc.get(id_field, ''))
highlighted_fields = response.get('highlighting', {}).keys()
if highlighted_fields and not query.get('download'):
id_field = collection.get('idField')
if id_field:
for doc in response['response']['docs']:
if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields:
highlighting = response['highlighting'][smart_unicode(doc[id_field])]
if highlighting:
escaped_highlighting = {}
for field, hls in highlighting.iteritems():
_hls = [escape(smart_unicode(hl, errors='replace')).replace('<em>', '<em>').replace('</em>', '</em>') for hl in hls]
escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls
doc.update(escaped_highlighting)
else:
response['warning'] = _("The Solr schema requires an id field for performing the result highlighting")
if normalized_facets:
augmented['normalized_facets'].extend(normalized_facets)
return augmented
def augment_solr_response(response, collection, query):
augmented = response
augmented['normalized_facets'] = []
NAME = '%(field)s-%(id)s'
normalized_facets = []
selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
if response and response.get('facet_counts'):
for facet in collection['facets']:
category = facet['type']
if category == 'field' and response['facet_counts']['facet_fields']:
name = NAME % facet
collection_facet = get_facet_field(category, name,
collection['facets'])
counts = pairwise2(
facet['field'], selected_values.get(facet['id'], []),
response['facet_counts']['facet_fields'][name])
if collection_facet['properties']['sort'] == 'asc':
counts.reverse()
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
}
normalized_facets.append(facet)
elif (category == 'range' or category
== 'range-up') and response['facet_counts']['facet_ranges']:
name = NAME % facet
collection_facet = get_facet_field(category, name,
collection['facets'])
counts = response['facet_counts']['facet_ranges'][name][
'counts']
end = response['facet_counts']['facet_ranges'][name]['end']
counts = range_pair(facet['field'], name,
selected_values.get(facet['id'], []),
counts, end, collection_facet)
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
'extraSeries': []
}
normalized_facets.append(facet)
elif category == 'query' and response['facet_counts'][
'facet_queries']:
for name, value in response['facet_counts'][
'facet_queries'].iteritems():
collection_facet = get_facet_field(category, name,
collection['facets'])
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': value,
}
normalized_facets.append(facet)
elif category == 'pivot':
name = NAME % facet
if 'facet_pivot' in response[
'facet_counts'] and name in response['facet_counts'][
'facet_pivot']:
if facet['properties']['scope'] == 'stack':
count = _augment_pivot_2d(
name, facet['id'],
response['facet_counts']['facet_pivot'][name],
selected_values)
else:
count = response['facet_counts']['facet_pivot'][name]
_augment_pivot_nd(facet['id'], count, selected_values)
else:
count = []
facet = {
'id': facet['id'],
'field': name,
'type': category,
'label': name,
'counts': count,
}
normalized_facets.append(facet)
if response and response.get('facets'):
for facet in collection['facets']:
category = facet['type']
name = facet['id'] # Nested facets can only have one name
if category == 'function' and name in response['facets']:
collection_facet = get_facet_field(category, name,
collection['facets'])
value = response['facets'][name]
counts = {'value': value, 'percentage': 0}
if collection_facet['properties']['filter']['is_enabled']:
if collection_facet['properties']['compare']['is_enabled']:
value = value[name]
else:
counts['value'] = value['count']
if collection_facet['properties']['compare']['is_enabled']:
orignal_number, final_number = value['buckets'][0].get(
name, 0), value['buckets'][1].get(name, 0)
if collection_facet['properties']['compare'][
'use_percentage']:
if orignal_number != 0:
counts['percentage'] = (
final_number -
orignal_number) / orignal_number * 100.0
counts['value'] = final_number - orignal_number
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': counts,
}
normalized_facets.append(facet)
elif category == 'nested' and name in response['facets']:
value = response['facets'][name]
collection_facet = get_facet_field(category, name,
collection['facets'])
extraSeries = []
counts = response['facets'][name]['buckets']
# Give humane names to the columns
cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
last_seen_dim_col_index = 0
prev_last_seen_dim_col_index = 0
for i, f in enumerate(facet['properties']['facets'][1:]):
if f['aggregate']['function'] == 'count':
cols.append(f['field'])
prev_last_seen_dim_col_index = last_seen_dim_col_index
last_seen_dim_col_index = i + 2
from libsolr.api import SolrApi
aggregate_name = SolrApi._get_aggregate_function(f)
cols.append(aggregate_name +
('_%(field)s' % facet['properties']['facets']
[last_seen_dim_col_index -
1] if aggregate_name in cols else ''))
rows = []
facet_one = collection_facet['properties']['facets'][0]
if 'missing' in value:
counts.append({
'val': '',
'count': value['missing']['count']
})
# Number or Date range
if facet_one['canRange'] and not facet_one['type'] == 'field':
dimension = 3 if facet_one['isDate'] else 1
# Single dimension or dimension 2 with analytics
if len(collection_facet['properties']
['facets']) == 1 or len(
collection_facet['properties']['facets']
) == 2 and collection_facet['properties']['facets'][
1]['aggregate']['function'] != 'count':
column = 'count'
if len(collection_facet['properties']['facets']) == 2:
agg_keys = [
key for key, value in counts[0].items()
if key.lower().startswith('agg_')
]
legend = agg_keys[0].split(':', 2)[1]
column = agg_keys[0]
else:
legend = facet['field'] # 'count(%s)' % legend
agg_keys = [column]
_augment_stats_2d(name, facet, counts, selected_values,
agg_keys, rows)
counts = [
_v for _f in counts
for _v in (_f['val'], _f[column])
]
counts = range_pair2(
facet['field'],
name,
selected_values.get(facet['id'], []),
counts,
1,
collection_facet['properties']['facets'][0],
collection_facet=collection_facet)
else:
# Dimension 1 with counts and 2 with analytics
agg_keys = [
key for key, value in counts[0].items()
if key.lower().startswith('agg_')
or key.lower().startswith('dim_')
]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower(
).startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts,
selected_values, agg_keys,
rows)
_series = collections.defaultdict(list)
for row in rows:
for i, cell in enumerate(row):
if i > last_seen_dim_col_index:
legend = cols[i]
if prev_last_seen_dim_col_index != last_seen_dim_col_index:
legend = '%s %s' % (
cols[i],
row[last_seen_dim_col_index])
_series[legend].append(
row[prev_last_seen_dim_col_index])
_series[legend].append(cell)
for _name, val in _series.iteritems():
_c = range_pair2(
facet['field'], _name,
selected_values.get(facet['id'], []), val, 1,
collection_facet['properties']['facets'][0])
extraSeries.append({'counts': _c, 'label': _name})
counts = []
elif collection_facet['properties'].get('isOldPivot'):
facet_fields = [collection_facet['field']] + [
f['field'] for f in collection_facet['properties'].get(
'facets', [])
if f['aggregate']['function'] == 'count'
]
column = 'count'
agg_keys = [
key for key, value in counts[0].items()
if key.lower().startswith('agg_')
or key.lower().startswith('dim_')
]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith(
'dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts,
selected_values, agg_keys, rows)
#_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
dimension = len(facet_fields)
elif len(collection_facet['properties']['facets']) == 1 or (
len(collection_facet['properties']['facets']) == 2
and collection_facet['properties']['facets'][1]
['aggregate']['function'] != 'count'):
# Dimension 1 with 1 count or agg
dimension = 1
column = 'count'
agg_keys = counts and [
key for key, value in counts[0].items()
if key.lower().startswith('agg_')
]
if len(collection_facet['properties']
['facets']) == 2 and agg_keys:
column = agg_keys[0]
else:
agg_keys = [column]
legend = collection_facet['properties']['facets'][0][
'field']
_augment_stats_2d(name, facet, counts, selected_values,
agg_keys, rows)
counts = [
_v for _f in counts for _v in (_f['val'], _f[column])
]
counts = pairwise2(legend,
selected_values.get(facet['id'], []),
counts)
else:
# Dimension 2 with analytics or 1 with N aggregates
dimension = 2
agg_keys = counts and [
key for key, value in counts[0].items()
if key.lower().startswith('agg_')
or key.lower().startswith('dim_')
]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith(
'dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts,
selected_values, agg_keys, rows)
actual_dimension = sum([
_f['aggregate']['function'] == 'count'
for _f in collection_facet['properties']['facets']
])
counts = filter(
lambda a: len(a['fq_fields']) == actual_dimension,
counts)
num_bucket = response['facets'][name][
'numBuckets'] if 'numBuckets' in response['facets'][
name] else len(response['facets'][name])
facet = {
'id':
collection_facet['id'],
'field':
facet['field'],
'type':
category,
'label':
collection_facet['label'],
'counts':
counts,
'extraSeries':
extraSeries,
'dimension':
dimension,
'response': {
'response': {
'start': 0,
'numFound': num_bucket
}
}, # Todo * nested buckets + offsets
'docs': [dict(zip(cols, row)) for row in rows],
'fieldsAttributes': [
Collection2._make_gridlayout_header_field({
'name':
col,
'type':
'aggr' if '(' in col else 'string'
}) for col in cols
]
}
normalized_facets.append(facet)
# Remove unnecessary facet data
if response:
response.pop('facet_counts')
response.pop('facets')
augment_response(collection, query, response)
if normalized_facets:
augmented['normalized_facets'].extend(normalized_facets)
return augmented
def augment_solr_response(response, collection, query):
augmented = response
augmented['normalized_facets'] = []
NAME = '%(field)s-%(id)s'
normalized_facets = []
selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']])
if response and response.get('facet_counts'):
for facet in collection['facets']:
category = facet['type']
if category == 'field' and response['facet_counts']['facet_fields']:
name = NAME % facet
collection_facet = get_facet_field(category, name, collection['facets'])
counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name])
if collection_facet['properties']['sort'] == 'asc':
counts.reverse()
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
}
normalized_facets.append(facet)
elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']:
name = NAME % facet
collection_facet = get_facet_field(category, name, collection['facets'])
counts = response['facet_counts']['facet_ranges'][name]['counts']
end = response['facet_counts']['facet_ranges'][name]['end']
counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet)
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
'extraSeries': []
}
normalized_facets.append(facet)
elif category == 'query' and response['facet_counts']['facet_queries']:
for name, value in response['facet_counts']['facet_queries'].iteritems():
collection_facet = get_facet_field(category, name, collection['facets'])
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': value,
}
normalized_facets.append(facet)
elif category == 'pivot':
name = NAME % facet
if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']:
if facet['properties']['scope'] == 'stack':
count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values)
else:
count = response['facet_counts']['facet_pivot'][name]
_augment_pivot_nd(facet['id'], count, selected_values)
else:
count = []
facet = {
'id': facet['id'],
'field': name,
'type': category,
'label': name,
'counts': count,
}
normalized_facets.append(facet)
if response and response.get('facets'):
for facet in collection['facets']:
category = facet['type']
name = facet['id'] # Nested facets can only have one name
if category == 'function' and name in response['facets']:
value = response['facets'][name]
collection_facet = get_facet_field(category, name, collection['facets'])
facet = {
'id': collection_facet['id'],
'query': name,
'type': category,
'label': name,
'counts': value,
}
normalized_facets.append(facet)
elif category == 'nested' and name in response['facets']:
value = response['facets'][name]
collection_facet = get_facet_field(category, name, collection['facets'])
extraSeries = []
counts = response['facets'][name]['buckets']
cols = ['%(field)s' % facet, 'count(%(field)s)' % facet]
last_x_col = 0
last_xx_col = 0
for i, f in enumerate(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
cols.append(f['field'])
last_xx_col = last_x_col
last_x_col = i + 2
cols.append(SolrApi._get_aggregate_function(f))
rows = []
# For dim in dimensions
# Number or Date range
if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field':
dimension = 3 if collection_facet['properties']['isDate'] else 1
# Single dimension or dimension 2 with analytics
if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1:
column = 'count'
if len(collection_facet['properties']['facets']) == 1:
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
legend = agg_keys[0].split(':', 2)[1]
column = agg_keys[0]
else:
legend = facet['field'] # 'count(%s)' % legend
agg_keys = [column]
_augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet)
else:
# Dimension 1 with counts and 2 with analytics
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
_series = collections.defaultdict(list)
for row in rows:
for i, cell in enumerate(row):
if i > last_x_col:
legend = cols[i]
if last_xx_col != last_x_col:
legend = '%s %s' % (cols[i], row[last_x_col])
_series[legend].append(row[last_xx_col])
_series[legend].append(cell)
for name, val in _series.iteritems():
_c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet)
extraSeries.append({'counts': _c, 'label': name})
counts = []
elif collection_facet['properties'].get('isOldPivot'):
facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count']
column = 'count'
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
#_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
dimension = len(facet_fields)
elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
# Dimension 1 with 1 count or agg
dimension = 1
column = 'count'
if len(collection_facet['properties']['facets']) == 1:
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')]
legend = agg_keys[0].split(':', 2)[1]
column = agg_keys[0]
else:
legend = facet['field']
agg_keys = [column]
_augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
counts = [_v for _f in counts for _v in (_f['val'], _f[column])]
counts = pairwise2(legend, selected_values.get(facet['id'], []), counts)
else:
# Dimension 2 with analytics or 1 with N aggregates
dimension = 2
agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')]
agg_keys.sort(key=lambda a: a[4:])
if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
agg_keys.insert(0, 'count')
counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']])
counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts)
num_bucket = response['facets'][name]['numBuckets'] if 'numBuckets' in response['facets'][name] else len(response['facets'][name])
facet = {
'id': collection_facet['id'],
'field': facet['field'],
'type': category,
'label': collection_facet['label'],
'counts': counts,
'extraSeries': extraSeries,
'dimension': dimension,
'response': {'response': {'start': 0, 'numFound': num_bucket}}, # Todo * nested buckets + offsets
'docs': [dict(zip(cols, row)) for row in rows],
'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols]
}
normalized_facets.append(facet)
# Remove unnecessary facet data
if response:
response.pop('facet_counts')
response.pop('facets')
# HTML escaping
if not query.get('download'):
id_field = collection.get('idField', '')
for doc in response['response']['docs']:
for field, value in doc.iteritems():
if isinstance(value, numbers.Number):
escaped_value = value
elif field == '_childDocuments_': # Nested documents
escaped_value = value
elif isinstance(value, list): # Multivalue field
escaped_value = [smart_unicode(escape(val), errors='replace') for val in value]
else:
value = smart_unicode(value, errors='replace')
escaped_value = escape(value)
doc[field] = escaped_value
link = None
if 'link-meta' in doc:
meta = json.loads(doc['link-meta'])
link = get_data_link(meta)
doc['externalLink'] = link
doc['details'] = []
doc['hueId'] = smart_unicode(doc.get(id_field, ''))
highlighted_fields = response.get('highlighting', {}).keys()
if highlighted_fields and not query.get('download'):
id_field = collection.get('idField')
if id_field:
for doc in response['response']['docs']:
if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields:
highlighting = response['highlighting'][smart_unicode(doc[id_field])]
if highlighting:
escaped_highlighting = {}
for field, hls in highlighting.iteritems():
_hls = [escape(smart_unicode(hl, errors='replace')).replace('<em>', '<em>').replace('</em>', '</em>') for hl in hls]
escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls
doc.update(escaped_highlighting)
else:
response['warning'] = _("The Solr schema requires an id field for performing the result highlighting")
if normalized_facets:
augmented['normalized_facets'].extend(normalized_facets)
return augmented
