def query_ec(self, str_query, q_fields, start_date=0, end_date=0, index='logs-*', doc_type='logs', hours=24, debug=False): if start_date > end_date: raise Exception('The start_date can\'t be greater than the end_date') if start_date == 0 or end_date == 0: dt_end_date = datetime.now().timestamp() dt_start_date = (datetime.now() - timedelta(hours=hours)).timestamp() start_date = int(dt_start_date) * 1000 end_date = int(dt_end_date) * 1000 # print(str(start_date) + ' -- ' + str(end_date)) elastic_qry = ElasticQuery(es=self.elastic_client, index=index, doc_type=doc_type) elastic_qry.query( Query.bool( must=[Query.query_string(str_query), Query.range('normalDate', gte=start_date, lte=end_date)] ) ) elastic_qry.aggregate( Aggregate.date_histogram('2', 'normalDate', '12h') ) my_qry = elastic_qry.dict() my_qry['stored_fields'] = q_fields search_arr = list() header_qry = {"index": ["logs-*"], "ignore_unavailable": True} search_arr.append(header_qry) search_arr.append(my_qry) print('Elastic Query: ' + str(search_arr)) print('------------------------------------------------------------------------------------') print('Lucene Query: ' + str_query) request = '' for each in search_arr: request += '%s \n' % json.dumps(each) # print(request) resp = self.elastic_client.msearch(body=request) if resp is None and len(resp['responses']) <= 0: return None else: response = resp['responses'][0] hits_data = list() if response['hits']['total'] > 0: for hit in response['hits']['hits']: hits_data.append(hit) # print(str(hits_data)) return search_arr, hits_data
def test_invalid_arg(self): # Test passing not a list with self.assertRaises(ValueError): Query.bool(must=set()) # And now an invalid list with self.assertRaises(ValueError): Query.bool(must=[None]) # And now an invalid list with self.assertRaises(ValueError): Query.bool(must=[Aggregate.terms('test', 'test')]) # And now an invalid list with self.assertRaises(ValueError): Query.range('field', gte=['error']) # Empty list should be OK/ignored Query.bool(must=[])
def search(self, parms): device_controller = DeviceController(self.db, self.logger) if "location" in parms.keys(): list_location_ip = device_controller.get_device_list_by_locationid( parms["location"]) if "device" in parms.keys(): list_device_ip = device_controller.get_device_list_by_hostname( parms["device"]) #Doing intersection between device search and device in location search_ip = list(set(list_location_ip) & set(list_device_ip)) print search_ip query_search = [] if search_ip: query_search.append(Query.terms('host', search_ip)) if parms["time"]: time_from = parms["time"]["from"].split(" ")[0] time_to = parms["time"]["to"].split(" ")[0] query_search.append( Query.range('@timestamp', gte=time_from, lte=time_to)) if parms["severityLevel"]: query_search.append( Query.terms('severity', parms["severityLevel"])) if parms["keywordMessage"]: message_search = [] message_search.append(parms["keywordMessage"]) query_search.append(Query.terms('message', message_search)) index = "syslog*" es = Elasticsearch(["http://192.168.100.249:9200"]) q = ElasticQuery(es=es, index=index, doc_type='doc') # q.query(Query.match_all()) q.size(1000) q.query(Query.bool(must=query_search)) #q.query(Aggregate.terms(must=query_search)) print q.json(indent=4) query_result = self.format_results(q.get()) return query_result #No index to query else: return []
'default_operator': 'AND' } }, 'STRING': { 'query_string': { 'query': 'field_name1:value_name1 AND (field_name2:((value_name2) OR (value_name3)))', 'default_operator': 'AND' } }, 'NESTED_FILTER': {} } # Test filters print '[ElasticQuery] Testing: filters & queries' query = Query.range('field_name1', gt=0, lte=100)[1] test('Query.range', query, FILTERS['RANGE']) query = Query.prefix(field_name1='value_name1')[1] test('Query.prefix', query, FILTERS['PREFIX']) query = Query.term(field_name1='value_name1')[1] test('Query.term', query, FILTERS['TERM']) query = Query.terms(field_name1=['value_name1', 'value_name2'])[1] test('Query.terms', query, FILTERS['TERMS']) query = Filter.missing('field_name1')[1] test('Filter.missing', query, FILTERS['FILTER_MISSING']) query = Query.missing('field_name1')[1] test('Query.missing', query, FILTERS['QUERY_MISSING'])