def augment_solr_response(response, collection, query): augmented = response augmented['normalized_facets'] = [] NAME = '%(field)s-%(id)s' normalized_facets = [] selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']]) if response and response.get('facet_counts'): for facet in collection['facets']: category = facet['type'] if category == 'field' and response['facet_counts']['facet_fields']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name]) if collection_facet['properties']['sort'] == 'asc': counts.reverse() facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, } normalized_facets.append(facet) elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = response['facet_counts']['facet_ranges'][name]['counts'] end = response['facet_counts']['facet_ranges'][name]['end'] counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': [] } normalized_facets.append(facet) elif category == 'query' and response['facet_counts']['facet_queries']: for name, value in response['facet_counts']['facet_queries'].iteritems(): collection_facet = get_facet_field(category, name, collection['facets']) facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': value, } normalized_facets.append(facet) elif category == 'pivot': name = NAME % facet if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']: if facet['properties']['scope'] == 'stack': count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values) else: count = response['facet_counts']['facet_pivot'][name] _augment_pivot_nd(facet['id'], count, selected_values) else: count = [] facet = { 'id': facet['id'], 'field': name, 'type': category, 'label': name, 'counts': count, } normalized_facets.append(facet) if response and response.get('facets'): for facet in collection['facets']: category = facet['type'] name = facet['id'] # Nested facets can only have one name if category == 'function' and name in response['facets']: value = response['facets'][name] collection_facet = get_facet_field(category, name, collection['facets']) facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': value, } normalized_facets.append(facet) elif category == 'nested' and name in response['facets']: value = response['facets'][name] collection_facet = get_facet_field(category, name, collection['facets']) extraSeries = [] counts = response['facets'][name]['buckets'] cols = ['%(field)s' % facet, 'count(%(field)s)' % facet] last_x_col = 0 last_xx_col = 0 for i, f in enumerate(facet['properties']['facets']): if f['aggregate']['function'] == 'count': cols.append(f['field']) last_xx_col = last_x_col last_x_col = i + 2 cols.append(SolrApi._get_aggregate_function(f)) rows = [] # For dim in dimensions # Number or Date range if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field': dimension = 3 # Single dimension or dimension 2 with analytics if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1: column = 'count' if len(collection_facet['properties']['facets']) == 1: agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')] legend = agg_keys[0].split(':', 2)[1] column = agg_keys[0] else: legend = facet['field'] # 'count(%s)' % legend agg_keys = [column] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [_v for _f in counts for _v in (_f['val'], _f[column])] counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet) else: # Dimension 1 with counts and 2 with analytics agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) _series = collections.defaultdict(list) for row in rows: for i, cell in enumerate(row): if i > last_x_col: legend = cols[i] if last_xx_col != last_x_col: legend = '%s %s' % (cols[i], row[last_x_col]) _series[legend].append(row[last_xx_col]) _series[legend].append(cell) for name, val in _series.iteritems(): _c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet) extraSeries.append({'counts': _c, 'label': name}) counts = [] elif collection_facet['properties'].get('isOldPivot'): facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count'] column = 'count' agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) # _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) print counts # count = response['facets'][name] # _convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2) dimension = len(facet_fields) elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1): # Dimension 1 with 1 count or agg dimension = 1 column = 'count' if len(collection_facet['properties']['facets']) == 1: agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')] legend = agg_keys[0].split(':', 2)[1] column = agg_keys[0] else: legend = facet['field'] agg_keys = [column] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [_v for _f in counts for _v in (_f['val'], _f[column])] counts = pairwise2(legend, selected_values.get(facet['id'], []), counts) else: # Dimension 2 with analytics or 1 with N aggregates dimension = 2 agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']]) counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': extraSeries, 'dimension': dimension, 'response': {'response': {'start': 0, 'numFound': response['facets'][name]['numBuckets']}}, # Todo * nested buckets + offsets 'docs': [dict(zip(cols, row)) for row in rows], 'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols] } normalized_facets.append(facet) # Remove unnecessary facet data if response: response.pop('facet_counts') response.pop('facets') # HTML escaping if not query.get('download'): id_field = collection.get('idField', '') for doc in response['response']['docs']: for field, value in doc.iteritems(): if isinstance(value, numbers.Number): escaped_value = value elif field == '_childDocuments_': # Nested documents escaped_value = value elif isinstance(value, list): # Multivalue field escaped_value = [smart_unicode(escape(val), errors='replace') for val in value] else: value = smart_unicode(value, errors='replace') escaped_value = escape(value) doc[field] = escaped_value link = None if 'link-meta' in doc: meta = json.loads(doc['link-meta']) link = get_data_link(meta) doc['externalLink'] = link doc['details'] = [] doc['hueId'] = smart_unicode(doc.get(id_field, '')) highlighted_fields = response.get('highlighting', {}).keys() if highlighted_fields and not query.get('download'): id_field = collection.get('idField') if id_field: for doc in response['response']['docs']: if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields: highlighting = response['highlighting'][smart_unicode(doc[id_field])] if highlighting: escaped_highlighting = {} for field, hls in highlighting.iteritems(): _hls = [escape(smart_unicode(hl, errors='replace')).replace('<em>', '<em>').replace('</em>', '</em>') for hl in hls] escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls doc.update(escaped_highlighting) else: response['warning'] = _("The Solr schema requires an id field for performing the result highlighting") if normalized_facets: augmented['normalized_facets'].extend(normalized_facets) return augmented
def augment_solr_response(response, collection, query): augmented = response augmented['normalized_facets'] = [] NAME = '%(field)s-%(id)s' normalized_facets = [] selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']]) if response and response.get('facet_counts'): for facet in collection['facets']: category = facet['type'] if category == 'field' and response['facet_counts']['facet_fields']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = pairwise2( facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name]) if collection_facet['properties']['sort'] == 'asc': counts.reverse() facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, } normalized_facets.append(facet) elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = response['facet_counts']['facet_ranges'][name][ 'counts'] end = response['facet_counts']['facet_ranges'][name]['end'] counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': [] } normalized_facets.append(facet) elif category == 'query' and response['facet_counts'][ 'facet_queries']: for name, value in response['facet_counts'][ 'facet_queries'].iteritems(): collection_facet = get_facet_field(category, name, collection['facets']) facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': value, } normalized_facets.append(facet) elif category == 'pivot': name = NAME % facet if 'facet_pivot' in response[ 'facet_counts'] and name in response['facet_counts'][ 'facet_pivot']: if facet['properties']['scope'] == 'stack': count = _augment_pivot_2d( name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values) else: count = response['facet_counts']['facet_pivot'][name] _augment_pivot_nd(facet['id'], count, selected_values) else: count = [] facet = { 'id': facet['id'], 'field': name, 'type': category, 'label': name, 'counts': count, } normalized_facets.append(facet) if response and response.get('facets'): for facet in collection['facets']: category = facet['type'] name = facet['id'] # Nested facets can only have one name if category == 'function' and name in response['facets']: collection_facet = get_facet_field(category, name, collection['facets']) value = response['facets'][name] counts = {'value': value, 'percentage': 0} if collection_facet['properties']['filter']['is_enabled']: if collection_facet['properties']['compare']['is_enabled']: value = value[name] else: counts['value'] = value['count'] if collection_facet['properties']['compare']['is_enabled']: orignal_number, final_number = value['buckets'][0].get( name, 0), value['buckets'][1].get(name, 0) if collection_facet['properties']['compare'][ 'use_percentage']: if orignal_number != 0: counts['percentage'] = ( final_number - orignal_number) / orignal_number * 100.0 counts['value'] = final_number - orignal_number facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': counts, } normalized_facets.append(facet) elif category == 'nested' and name in response['facets']: value = response['facets'][name] collection_facet = get_facet_field(category, name, collection['facets']) extraSeries = [] counts = response['facets'][name]['buckets'] # Give humane names to the columns cols = ['%(field)s' % facet, 'count(%(field)s)' % facet] last_seen_dim_col_index = 0 prev_last_seen_dim_col_index = 0 for i, f in enumerate(facet['properties']['facets'][1:]): if f['aggregate']['function'] == 'count': cols.append(f['field']) prev_last_seen_dim_col_index = last_seen_dim_col_index last_seen_dim_col_index = i + 2 from libsolr.api import SolrApi aggregate_name = SolrApi._get_aggregate_function(f) cols.append(aggregate_name + ('_%(field)s' % facet['properties']['facets'] [last_seen_dim_col_index - 1] if aggregate_name in cols else '')) rows = [] facet_one = collection_facet['properties']['facets'][0] if 'missing' in value: counts.append({ 'val': '', 'count': value['missing']['count'] }) # Number or Date range if facet_one['canRange'] and not facet_one['type'] == 'field': dimension = 3 if facet_one['isDate'] else 1 # Single dimension or dimension 2 with analytics if len(collection_facet['properties'] ['facets']) == 1 or len( collection_facet['properties']['facets'] ) == 2 and collection_facet['properties']['facets'][ 1]['aggregate']['function'] != 'count': column = 'count' if len(collection_facet['properties']['facets']) == 2: agg_keys = [ key for key, value in counts[0].items() if key.lower().startswith('agg_') ] legend = agg_keys[0].split(':', 2)[1] column = agg_keys[0] else: legend = facet['field'] # 'count(%s)' % legend agg_keys = [column] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [ _v for _f in counts for _v in (_f['val'], _f[column]) ] counts = range_pair2( facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet['properties']['facets'][0], collection_facet=collection_facet) else: # Dimension 1 with counts and 2 with analytics agg_keys = [ key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_') ] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower( ).startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) _series = collections.defaultdict(list) for row in rows: for i, cell in enumerate(row): if i > last_seen_dim_col_index: legend = cols[i] if prev_last_seen_dim_col_index != last_seen_dim_col_index: legend = '%s %s' % ( cols[i], row[last_seen_dim_col_index]) _series[legend].append( row[prev_last_seen_dim_col_index]) _series[legend].append(cell) for _name, val in _series.iteritems(): _c = range_pair2( facet['field'], _name, selected_values.get(facet['id'], []), val, 1, collection_facet['properties']['facets'][0]) extraSeries.append({'counts': _c, 'label': _name}) counts = [] elif collection_facet['properties'].get('isOldPivot'): facet_fields = [collection_facet['field']] + [ f['field'] for f in collection_facet['properties'].get( 'facets', []) if f['aggregate']['function'] == 'count' ] column = 'count' agg_keys = [ key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_') ] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith( 'dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2) dimension = len(facet_fields) elif len(collection_facet['properties']['facets']) == 1 or ( len(collection_facet['properties']['facets']) == 2 and collection_facet['properties']['facets'][1] ['aggregate']['function'] != 'count'): # Dimension 1 with 1 count or agg dimension = 1 column = 'count' agg_keys = counts and [ key for key, value in counts[0].items() if key.lower().startswith('agg_') ] if len(collection_facet['properties'] ['facets']) == 2 and agg_keys: column = agg_keys[0] else: agg_keys = [column] legend = collection_facet['properties']['facets'][0][ 'field'] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [ _v for _f in counts for _v in (_f['val'], _f[column]) ] counts = pairwise2(legend, selected_values.get(facet['id'], []), counts) else: # Dimension 2 with analytics or 1 with N aggregates dimension = 2 agg_keys = counts and [ key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_') ] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith( 'dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) actual_dimension = sum([ _f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets'] ]) counts = filter( lambda a: len(a['fq_fields']) == actual_dimension, counts) num_bucket = response['facets'][name][ 'numBuckets'] if 'numBuckets' in response['facets'][ name] else len(response['facets'][name]) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': extraSeries, 'dimension': dimension, 'response': { 'response': { 'start': 0, 'numFound': num_bucket } }, # Todo * nested buckets + offsets 'docs': [dict(zip(cols, row)) for row in rows], 'fieldsAttributes': [ Collection2._make_gridlayout_header_field({ 'name': col, 'type': 'aggr' if '(' in col else 'string' }) for col in cols ] } normalized_facets.append(facet) # Remove unnecessary facet data if response: response.pop('facet_counts') response.pop('facets') augment_response(collection, query, response) if normalized_facets: augmented['normalized_facets'].extend(normalized_facets) return augmented
def augment_solr_response(response, collection, query): augmented = response augmented['normalized_facets'] = [] NAME = '%(field)s-%(id)s' normalized_facets = [] selected_values = dict([(fq['id'], fq['filter']) for fq in query['fqs']]) if response and response.get('facet_counts'): for facet in collection['facets']: category = facet['type'] if category == 'field' and response['facet_counts']['facet_fields']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = pairwise2(facet['field'], selected_values.get(facet['id'], []), response['facet_counts']['facet_fields'][name]) if collection_facet['properties']['sort'] == 'asc': counts.reverse() facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, } normalized_facets.append(facet) elif (category == 'range' or category == 'range-up') and response['facet_counts']['facet_ranges']: name = NAME % facet collection_facet = get_facet_field(category, name, collection['facets']) counts = response['facet_counts']['facet_ranges'][name]['counts'] end = response['facet_counts']['facet_ranges'][name]['end'] counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, end, collection_facet) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': [] } normalized_facets.append(facet) elif category == 'query' and response['facet_counts']['facet_queries']: for name, value in response['facet_counts']['facet_queries'].iteritems(): collection_facet = get_facet_field(category, name, collection['facets']) facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': value, } normalized_facets.append(facet) elif category == 'pivot': name = NAME % facet if 'facet_pivot' in response['facet_counts'] and name in response['facet_counts']['facet_pivot']: if facet['properties']['scope'] == 'stack': count = _augment_pivot_2d(name, facet['id'], response['facet_counts']['facet_pivot'][name], selected_values) else: count = response['facet_counts']['facet_pivot'][name] _augment_pivot_nd(facet['id'], count, selected_values) else: count = [] facet = { 'id': facet['id'], 'field': name, 'type': category, 'label': name, 'counts': count, } normalized_facets.append(facet) if response and response.get('facets'): for facet in collection['facets']: category = facet['type'] name = facet['id'] # Nested facets can only have one name if category == 'function' and name in response['facets']: value = response['facets'][name] collection_facet = get_facet_field(category, name, collection['facets']) facet = { 'id': collection_facet['id'], 'query': name, 'type': category, 'label': name, 'counts': value, } normalized_facets.append(facet) elif category == 'nested' and name in response['facets']: value = response['facets'][name] collection_facet = get_facet_field(category, name, collection['facets']) extraSeries = [] counts = response['facets'][name]['buckets'] cols = ['%(field)s' % facet, 'count(%(field)s)' % facet] last_x_col = 0 last_xx_col = 0 for i, f in enumerate(facet['properties']['facets']): if f['aggregate']['function'] == 'count': cols.append(f['field']) last_xx_col = last_x_col last_x_col = i + 2 cols.append(SolrApi._get_aggregate_function(f)) rows = [] # For dim in dimensions # Number or Date range if collection_facet['properties']['canRange'] and not facet['properties'].get('type') == 'field': dimension = 3 if collection_facet['properties']['isDate'] else 1 # Single dimension or dimension 2 with analytics if not collection_facet['properties']['facets'] or collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1: column = 'count' if len(collection_facet['properties']['facets']) == 1: agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')] legend = agg_keys[0].split(':', 2)[1] column = agg_keys[0] else: legend = facet['field'] # 'count(%s)' % legend agg_keys = [column] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [_v for _f in counts for _v in (_f['val'], _f[column])] counts = range_pair(facet['field'], name, selected_values.get(facet['id'], []), counts, 1, collection_facet) else: # Dimension 1 with counts and 2 with analytics agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) _series = collections.defaultdict(list) for row in rows: for i, cell in enumerate(row): if i > last_x_col: legend = cols[i] if last_xx_col != last_x_col: legend = '%s %s' % (cols[i], row[last_x_col]) _series[legend].append(row[last_xx_col]) _series[legend].append(cell) for name, val in _series.iteritems(): _c = range_pair(facet['field'], name, selected_values.get(facet['id'], []), val, 1, collection_facet) extraSeries.append({'counts': _c, 'label': name}) counts = [] elif collection_facet['properties'].get('isOldPivot'): facet_fields = [collection_facet['field']] + [f['field'] for f in collection_facet['properties'].get('facets', []) if f['aggregate']['function'] == 'count'] column = 'count' agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2) dimension = len(facet_fields) elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1): # Dimension 1 with 1 count or agg dimension = 1 column = 'count' if len(collection_facet['properties']['facets']) == 1: agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_')] legend = agg_keys[0].split(':', 2)[1] column = agg_keys[0] else: legend = facet['field'] agg_keys = [column] _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) counts = [_v for _f in counts for _v in (_f['val'], _f[column])] counts = pairwise2(legend, selected_values.get(facet['id'], []), counts) else: # Dimension 2 with analytics or 1 with N aggregates dimension = 2 agg_keys = [key for key, value in counts[0].items() if key.lower().startswith('agg_') or key.lower().startswith('dim_')] agg_keys.sort(key=lambda a: a[4:]) if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'): agg_keys.insert(0, 'count') counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows) actual_dimension = 1 + sum([_f['aggregate']['function'] == 'count' for _f in collection_facet['properties']['facets']]) counts = filter(lambda a: len(a['fq_fields']) == actual_dimension, counts) num_bucket = response['facets'][name]['numBuckets'] if 'numBuckets' in response['facets'][name] else len(response['facets'][name]) facet = { 'id': collection_facet['id'], 'field': facet['field'], 'type': category, 'label': collection_facet['label'], 'counts': counts, 'extraSeries': extraSeries, 'dimension': dimension, 'response': {'response': {'start': 0, 'numFound': num_bucket}}, # Todo * nested buckets + offsets 'docs': [dict(zip(cols, row)) for row in rows], 'fieldsAttributes': [Collection2._make_gridlayout_header_field({'name': col, 'type': 'aggr' if '(' in col else 'string'}) for col in cols] } normalized_facets.append(facet) # Remove unnecessary facet data if response: response.pop('facet_counts') response.pop('facets') # HTML escaping if not query.get('download'): id_field = collection.get('idField', '') for doc in response['response']['docs']: for field, value in doc.iteritems(): if isinstance(value, numbers.Number): escaped_value = value elif field == '_childDocuments_': # Nested documents escaped_value = value elif isinstance(value, list): # Multivalue field escaped_value = [smart_unicode(escape(val), errors='replace') for val in value] else: value = smart_unicode(value, errors='replace') escaped_value = escape(value) doc[field] = escaped_value link = None if 'link-meta' in doc: meta = json.loads(doc['link-meta']) link = get_data_link(meta) doc['externalLink'] = link doc['details'] = [] doc['hueId'] = smart_unicode(doc.get(id_field, '')) highlighted_fields = response.get('highlighting', {}).keys() if highlighted_fields and not query.get('download'): id_field = collection.get('idField') if id_field: for doc in response['response']['docs']: if id_field in doc and smart_unicode(doc[id_field]) in highlighted_fields: highlighting = response['highlighting'][smart_unicode(doc[id_field])] if highlighting: escaped_highlighting = {} for field, hls in highlighting.iteritems(): _hls = [escape(smart_unicode(hl, errors='replace')).replace('<em>', '<em>').replace('</em>', '</em>') for hl in hls] escaped_highlighting[field] = _hls[0] if len(_hls) == 1 else _hls doc.update(escaped_highlighting) else: response['warning'] = _("The Solr schema requires an id field for performing the result highlighting") if normalized_facets: augmented['normalized_facets'].extend(normalized_facets) return augmented