def __init__(self, request):
self.es_params = request.POST
self.ds = Datasets().activate_dataset(request.session)
self.index = self.ds.get_index()
self.mapping = self.ds.get_mapping()
self.es_m = ES_Manager(self.index, self.mapping)
self.field = 'texta_facts'
def __init__(self, es_index, es_mapping, field, query):
# Dataset info
self.es_index = es_index
self.es_mapping = es_mapping
self.field = field
# Build ES manager
self.es_m = ES_Manager(es_index, es_mapping)
self.es_m.load_combined_query(query)
def parse_request(self, request):
self.lookup_types = request.POST['lookup_types'].split(',')
self.key_constraints = request.POST['key_constraints'].split(',')
self.content = request.POST['content'].split('\n')[-1].strip()
print(self.content)
ds = Datasets().activate_dataset(request.session)
self.dataset = ds.get_index()
self.mapping = ds.get_mapping()
self.es_m = ES_Manager(self.dataset, self.mapping)
self.user = request.user
def __init__(self, request):
ds = Datasets().activate_dataset(request.session)
self.dataset = ds.get_index()
self.mapping = ds.get_mapping()
self.es_m = ES_Manager(self.dataset, self.mapping)
# PREPARE AGGREGATION
self.es_params = request.POST
interval = self.es_params["interval_1"]
self.daterange = self._get_daterange(self.es_params)
self.ranges, self.date_labels = self._get_date_intervals(
self.daterange, interval)
self.agg_query = self.prepare_agg_query()
# EXECUTE AGGREGATION
agg_results = self.aggregate()
# PARSE RESPONSES INTO JSON OBJECT
self.agg_data = self.parse_responses(agg_results)
def facts_agg(es_params, request):
logger = LogManager(__name__, 'FACTS AGGREGATION')
distinct_values = []
query_results = []
lexicon = []
aggregation_data = es_params['aggregate_over']
aggregation_data = json.loads(aggregation_data)
original_aggregation_field = aggregation_data['path']
aggregation_field = 'texta_link.facts'
try:
aggregation_size = 50
aggregations = {"strings": {es_params['sort_by']: {"field": aggregation_field, 'size': 0}},
"distinct_values": {"cardinality": {"field": aggregation_field}}}
# Define selected mapping
ds = Datasets().activate_dataset(request.session)
dataset = ds.get_index()
mapping = ds.get_mapping()
date_range = ds.get_date_range()
es_m = ES_Manager(dataset, mapping, date_range)
for item in es_params:
if 'saved_search' in item:
s = Search.objects.get(pk=es_params[item])
name = s.description
saved_query = json.loads(s.query)
es_m.load_combined_query(saved_query)
es_m.set_query_parameter('aggs', aggregations)
response = es_m.search()
# Filter response
bucket_filter = '{0}.'.format(original_aggregation_field.lower())
final_bucket = []
for b in response['aggregations']['strings']['buckets']:
if bucket_filter in b['key']:
fact_name = b['key'].split('.')[-1]
b['key'] = fact_name
final_bucket.append(b)
final_bucket = final_bucket[:aggregation_size]
response['aggregations']['distinct_values']['value'] = len(final_bucket)
response['aggregations']['strings']['buckets'] = final_bucket
normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings')
lexicon = list(set(lexicon+labels))
query_results.append({'name':name,'data':normalised_counts,'labels':labels})
distinct_values.append({'name':name,'data':response['aggregations']['distinct_values']['value']})
es_m.build(es_params)
# FIXME
# this is confusing for the user
if not es_m.is_combined_query_empty():
es_m.set_query_parameter('aggs', aggregations)
response = es_m.search()
# Filter response
bucket_filter = '{0}.'.format(original_aggregation_field.lower())
final_bucket = []
for b in response['aggregations']['strings']['buckets']:
if bucket_filter in b['key']:
fact_name = b['key'].split('.')[-1]
b['key'] = fact_name
final_bucket.append(b)
final_bucket = final_bucket[:aggregation_size]
response['aggregations']['distinct_values']['value'] = len(final_bucket)
response['aggregations']['strings']['buckets'] = final_bucket
normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings')
lexicon = list(set(lexicon+labels))
query_results.append({'name':'Query','data':normalised_counts,'labels':labels})
distinct_values.append({'name':'Query','data':response['aggregations']['distinct_values']['value']})
data = [a+zero_list(len(query_results)) for a in map(list, zip(*[lexicon]))]
data = [['Word']+[query_result['name'] for query_result in query_results]]+data
for i,word in enumerate(lexicon):
for j,query_result in enumerate(query_results):
for k,label in enumerate(query_result['labels']):
if word == label:
data[i+1][j+1] = query_result['data'][k]
logger.set_context('user_name', request.user.username)
logger.info('facts_aggregation_queried')
except Exception as e:
print('-- Exception[{0}] {1}'.format(__name__, e))
logger.set_context('user_name', request.user.username)
logger.exception('facts_aggregation_query_failed')
table_height = len(data)*15
table_height = table_height if table_height > 500 else 500
return {'data':[data[0]]+sorted(data[1:], key=lambda x: sum(x[1:]), reverse=True),'height':table_height,'type':'bar','distinct_values':json.dumps(distinct_values)}