def __init__(self, request): self.es_params = request.POST self.ds = Datasets().activate_dataset(request.session) self.index = self.ds.get_index() self.mapping = self.ds.get_mapping() self.es_m = ES_Manager(self.index, self.mapping) self.field = 'texta_facts'
def __init__(self, es_index, es_mapping, field, query): # Dataset info self.es_index = es_index self.es_mapping = es_mapping self.field = field # Build ES manager self.es_m = ES_Manager(es_index, es_mapping) self.es_m.load_combined_query(query)
def parse_request(self, request): self.lookup_types = request.POST['lookup_types'].split(',') self.key_constraints = request.POST['key_constraints'].split(',') self.content = request.POST['content'].split('\n')[-1].strip() print(self.content) ds = Datasets().activate_dataset(request.session) self.dataset = ds.get_index() self.mapping = ds.get_mapping() self.es_m = ES_Manager(self.dataset, self.mapping) self.user = request.user
def __init__(self, request): ds = Datasets().activate_dataset(request.session) self.dataset = ds.get_index() self.mapping = ds.get_mapping() self.es_m = ES_Manager(self.dataset, self.mapping) # PREPARE AGGREGATION self.es_params = request.POST interval = self.es_params["interval_1"] self.daterange = self._get_daterange(self.es_params) self.ranges, self.date_labels = self._get_date_intervals( self.daterange, interval) self.agg_query = self.prepare_agg_query() # EXECUTE AGGREGATION agg_results = self.aggregate() # PARSE RESPONSES INTO JSON OBJECT self.agg_data = self.parse_responses(agg_results)
def facts_agg(es_params, request): logger = LogManager(__name__, 'FACTS AGGREGATION') distinct_values = [] query_results = [] lexicon = [] aggregation_data = es_params['aggregate_over'] aggregation_data = json.loads(aggregation_data) original_aggregation_field = aggregation_data['path'] aggregation_field = 'texta_link.facts' try: aggregation_size = 50 aggregations = {"strings": {es_params['sort_by']: {"field": aggregation_field, 'size': 0}}, "distinct_values": {"cardinality": {"field": aggregation_field}}} # Define selected mapping ds = Datasets().activate_dataset(request.session) dataset = ds.get_index() mapping = ds.get_mapping() date_range = ds.get_date_range() es_m = ES_Manager(dataset, mapping, date_range) for item in es_params: if 'saved_search' in item: s = Search.objects.get(pk=es_params[item]) name = s.description saved_query = json.loads(s.query) es_m.load_combined_query(saved_query) es_m.set_query_parameter('aggs', aggregations) response = es_m.search() # Filter response bucket_filter = '{0}.'.format(original_aggregation_field.lower()) final_bucket = [] for b in response['aggregations']['strings']['buckets']: if bucket_filter in b['key']: fact_name = b['key'].split('.')[-1] b['key'] = fact_name final_bucket.append(b) final_bucket = final_bucket[:aggregation_size] response['aggregations']['distinct_values']['value'] = len(final_bucket) response['aggregations']['strings']['buckets'] = final_bucket normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings') lexicon = list(set(lexicon+labels)) query_results.append({'name':name,'data':normalised_counts,'labels':labels}) distinct_values.append({'name':name,'data':response['aggregations']['distinct_values']['value']}) es_m.build(es_params) # FIXME # this is confusing for the user if not es_m.is_combined_query_empty(): es_m.set_query_parameter('aggs', aggregations) response = es_m.search() # Filter response bucket_filter = '{0}.'.format(original_aggregation_field.lower()) final_bucket = [] for b in response['aggregations']['strings']['buckets']: if bucket_filter in b['key']: fact_name = b['key'].split('.')[-1] b['key'] = fact_name final_bucket.append(b) final_bucket = final_bucket[:aggregation_size] response['aggregations']['distinct_values']['value'] = len(final_bucket) response['aggregations']['strings']['buckets'] = final_bucket normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings') lexicon = list(set(lexicon+labels)) query_results.append({'name':'Query','data':normalised_counts,'labels':labels}) distinct_values.append({'name':'Query','data':response['aggregations']['distinct_values']['value']}) data = [a+zero_list(len(query_results)) for a in map(list, zip(*[lexicon]))] data = [['Word']+[query_result['name'] for query_result in query_results]]+data for i,word in enumerate(lexicon): for j,query_result in enumerate(query_results): for k,label in enumerate(query_result['labels']): if word == label: data[i+1][j+1] = query_result['data'][k] logger.set_context('user_name', request.user.username) logger.info('facts_aggregation_queried') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('user_name', request.user.username) logger.exception('facts_aggregation_query_failed') table_height = len(data)*15 table_height = table_height if table_height > 500 else 500 return {'data':[data[0]]+sorted(data[1:], key=lambda x: sum(x[1:]), reverse=True),'height':table_height,'type':'bar','distinct_values':json.dumps(distinct_values)}