def save(request): logger = LogManager(__name__, 'SAVE SEARCH') ds = Datasets().activate_dataset(request.session) es_m = ds.build_manager(ES_Manager) es_params = request.POST es_m.build(es_params) combined_query = es_m.get_combined_query() try: q = combined_query desc = request.POST['search_description'] s_content = json.dumps([request.POST[x] for x in request.POST.keys() if 'match_txt' in x]) search = Search(author=request.user,search_content=s_content,description=desc,dataset=Dataset.objects.get(pk=int(request.session['dataset'])),query=json.dumps(q)) search.save() logger.set_context('user_name', request.user.username) logger.set_context('search_id', search.id) logger.info('search_saved') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('es_params', es_params) logger.exception('search_saving_failed') return HttpResponse()
def _get_tags_for_text_index(self, text, alignment, highlight_data): logger = LogManager(__name__, '_GET_TAGS_FOR_TEXT_INDEX') data_mapping = {index: datum for index, datum in enumerate(highlight_data)} data_index_to_spans = [datum['spans'] for datum in highlight_data] text_index_to_data_index = [[] for i in range(len(text))] for data_index, spans in enumerate(data_index_to_spans): for span in spans: for text_index in range(*span): try: text_index_to_data_index[alignment[text_index]].append(data_index) except Exception as e: # Throws exception when index out of range # For example Gets index out of range if in _derive_highlight_data while uses >= instead of > print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('text', text) logger.exception('_get_tags_for_text_index try catch execption') text_index_to_data_index = [frozenset(data_indices) for data_indices in text_index_to_data_index] spans_to_tags = [(spans, self._get_tag_from_highlight_data([data_mapping[data_index] for data_index in data_indices])) for spans, data_indices in self._get_spans_to_data_indices(text_index_to_data_index)] return spans_to_tags
def remove_facts_from_document(self, rm_facts_dict, bs=7500): '''remove a certain fact from all documents given a [str]key and [str]val''' logger = LogManager(__name__, 'FACT MANAGER REMOVE FACTS') try: # Clears readonly block just in case the index has been set to read only self.es_m.clear_readonly_block() query = self._fact_deletion_query(rm_facts_dict) self.es_m.load_combined_query(query) response = self.es_m.scroll(size=bs, field_scroll=self.field) scroll_id = response['_scroll_id'] total_docs = response['hits']['total'] docs_left = total_docs # DEBUG print('Starting.. Total docs - ', total_docs) # DEBUG batch = 0 while total_docs > 0: print('Docs left:', docs_left) # DEBUG data = '' for document in response['hits']['hits']: new_field = [] # The new facts field for fact in document['_source'][self.field]: # If the fact name is in rm_facts_dict keys if fact["fact"] in rm_facts_dict: # If the fact value is not in the delete key values if fact['str_val'] not in rm_facts_dict.getlist( fact["fact"]): new_field.append(fact) else: new_field.append(fact) # Update dataset data += json.dumps({ "update": { "_id": document['_id'], "_type": document['_type'], "_index": document['_index'] } }) + '\n' document = {'doc': {self.field: new_field}} data += json.dumps(document) + '\n' response = self.es_m.scroll(scroll_id=scroll_id, size=bs, field_scroll=self.field) total_docs = len(response['hits']['hits']) docs_left -= bs # DEBUG scroll_id = response['_scroll_id'] self.es_m.plain_post_bulk(self.es_m.es_url, data) print('DONE') # DEBUG logger.set_context('docs_left', total_docs) logger.set_context('batch', batch) logger.info('remove_facts_from_document') except: print(traceback.format_exc()) logger.set_context('es_params', self.es_params) logger.exception('remove_facts_from_document_failed')
def delete(request): logger = LogManager(__name__, 'DELETE SEARCH') search_id = request.GET['pk'] logger.set_context('user_name', request.user.username) logger.set_context('search_id', search_id) try: Search.objects.get(pk=search_id).delete() logger.info('search_deleted') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.exception('search_deletion_failed') return HttpResponse(search_id)
def delete(request): post_data = json.loads(request.POST['data']) logger = LogManager(__name__, 'DELETE SEARCH') search_ids = post_data['pks'] logger.set_context('user_name', request.user.username) logger.set_context('search_ids', search_ids) try: for search_id in search_ids: Search.objects.get(pk=search_id).delete() logger.info('search_deleted:' + search_id) except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.exception('search_deletion_failed') return HttpResponse()
def delete(request): post_data = json.loads(request.POST['data']) logger = LogManager(__name__, 'DELETE SEARCH') search_ids = post_data['pks'] logger.set_context('user_name', request.user.username) logger.set_context('search_ids', search_ids) try: for search_id in search_ids: Search.objects.get(pk=search_id).delete() logger.info('search_deleted:' + search_id) except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.exception('search_deletion_failed') return HttpResponse()
def save(request): logger = LogManager(__name__, 'SAVE SEARCH') ds = Datasets().activate_datasets(request.session) es_m = ds.build_manager(ES_Manager) es_params = request.POST es_m.build(es_params) combined_query = es_m.get_combined_query() try: q = combined_query desc = request.POST['search_description'] s_content = {} # make json for x in request.POST.keys(): if 'match_txt' in x: # get the ID of the field, eg match_txt_1 returns 1 match_txt_1533 returns 1533 field_id = x.rsplit("_", 1)[-1] match_field = request.POST['match_field_' + field_id] if match_field in s_content.keys(): s_content[match_field].append(request.POST[x]) else: s_content[match_field] = [request.POST[x]] search = Search(author=request.user, search_content=json.dumps(s_content), description=desc, query=json.dumps(q)) logger.info('Saving search for datasets: {}'.format( request.session['dataset'])) search.save() for dataset_id in request.session['dataset']: dataset = Dataset.objects.get(pk=int(dataset_id)) search.datasets.add(dataset) search.save() logger.set_context('user_name', request.user.username) logger.set_context('search_id', search.id) logger.info('search_saved') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('es_params', es_params) logger.exception('search_saving_failed') return HttpResponse()
def save(request): logger = LogManager(__name__, 'SAVE SEARCH') ds = Datasets().activate_datasets(request.session) es_m = ds.build_manager(ES_Manager) es_params = request.POST es_m.build(es_params) combined_query = es_m.get_combined_query() try: q = combined_query desc = request.POST['search_description'] s_content = {} # make json for x in request.POST.keys(): if 'match_txt' in x: # get the ID of the field, eg match_txt_1 returns 1 match_txt_1533 returns 1533 field_id = x.rsplit("_", 1)[-1] match_field = request.POST['match_field_' + field_id] if match_field in s_content.keys(): s_content[match_field].append(request.POST[x]) else: s_content[match_field] = [request.POST[x]] search = Search(author=request.user, search_content=json.dumps(s_content), description=desc, query=json.dumps(q)) logger.info('Saving search for datasets: {}'.format(request.session['dataset'])) search.save() for dataset_id in request.session['dataset']: dataset = Dataset.objects.get(pk=int(dataset_id)) search.datasets.add(dataset) search.save() logger.set_context('user_name', request.user.username) logger.set_context('search_id', search.id) logger.info('search_saved') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('es_params', es_params) logger.exception('search_saving_failed') return HttpResponse()
def facts_agg(es_params, request): logger = LogManager(__name__, 'FACTS AGGREGATION') distinct_values = [] query_results = [] lexicon = [] aggregation_data = es_params['aggregate_over'] aggregation_data = json.loads(aggregation_data) original_aggregation_field = aggregation_data['path'] aggregation_field = 'texta_link.facts' try: aggregation_size = 50 aggregations = {"strings": {es_params['sort_by']: {"field": aggregation_field, 'size': 0}}, "distinct_values": {"cardinality": {"field": aggregation_field}}} # Define selected mapping ds = Datasets().activate_dataset(request.session) dataset = ds.get_index() mapping = ds.get_mapping() date_range = ds.get_date_range() es_m = ES_Manager(dataset, mapping, date_range) for item in es_params: if 'saved_search' in item: s = Search.objects.get(pk=es_params[item]) name = s.description saved_query = json.loads(s.query) es_m.load_combined_query(saved_query) es_m.set_query_parameter('aggs', aggregations) response = es_m.search() # Filter response bucket_filter = '{0}.'.format(original_aggregation_field.lower()) final_bucket = [] for b in response['aggregations']['strings']['buckets']: if bucket_filter in b['key']: fact_name = b['key'].split('.')[-1] b['key'] = fact_name final_bucket.append(b) final_bucket = final_bucket[:aggregation_size] response['aggregations']['distinct_values']['value'] = len(final_bucket) response['aggregations']['strings']['buckets'] = final_bucket normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings') lexicon = list(set(lexicon+labels)) query_results.append({'name':name,'data':normalised_counts,'labels':labels}) distinct_values.append({'name':name,'data':response['aggregations']['distinct_values']['value']}) es_m.build(es_params) # FIXME # this is confusing for the user if not es_m.is_combined_query_empty(): es_m.set_query_parameter('aggs', aggregations) response = es_m.search() # Filter response bucket_filter = '{0}.'.format(original_aggregation_field.lower()) final_bucket = [] for b in response['aggregations']['strings']['buckets']: if bucket_filter in b['key']: fact_name = b['key'].split('.')[-1] b['key'] = fact_name final_bucket.append(b) final_bucket = final_bucket[:aggregation_size] response['aggregations']['distinct_values']['value'] = len(final_bucket) response['aggregations']['strings']['buckets'] = final_bucket normalised_counts,labels = normalise_agg(response, es_m, es_params, 'strings') lexicon = list(set(lexicon+labels)) query_results.append({'name':'Query','data':normalised_counts,'labels':labels}) distinct_values.append({'name':'Query','data':response['aggregations']['distinct_values']['value']}) data = [a+zero_list(len(query_results)) for a in map(list, zip(*[lexicon]))] data = [['Word']+[query_result['name'] for query_result in query_results]]+data for i,word in enumerate(lexicon): for j,query_result in enumerate(query_results): for k,label in enumerate(query_result['labels']): if word == label: data[i+1][j+1] = query_result['data'][k] logger.set_context('user_name', request.user.username) logger.info('facts_aggregation_queried') except Exception as e: print('-- Exception[{0}] {1}'.format(__name__, e)) logger.set_context('user_name', request.user.username) logger.exception('facts_aggregation_query_failed') table_height = len(data)*15 table_height = table_height if table_height > 500 else 500 return {'data':[data[0]]+sorted(data[1:], key=lambda x: sum(x[1:]), reverse=True),'height':table_height,'type':'bar','distinct_values':json.dumps(distinct_values)}