def query_ec(self, str_query, q_fields, start_date=0, end_date=0, index='logs-*', doc_type='logs',
hours=24, debug=False):
if start_date > end_date:
raise Exception('The start_date can\'t be greater than the end_date')
if start_date == 0 or end_date == 0:
dt_end_date = datetime.now().timestamp()
dt_start_date = (datetime.now() - timedelta(hours=hours)).timestamp()
start_date = int(dt_start_date) * 1000
end_date = int(dt_end_date) * 1000
# print(str(start_date) + ' -- ' + str(end_date))
elastic_qry = ElasticQuery(es=self.elastic_client, index=index, doc_type=doc_type)
elastic_qry.query(
Query.bool(
must=[Query.query_string(str_query),
Query.range('normalDate', gte=start_date, lte=end_date)]
)
)
elastic_qry.aggregate(
Aggregate.date_histogram('2', 'normalDate', '12h')
)
my_qry = elastic_qry.dict()
my_qry['stored_fields'] = q_fields
search_arr = list()
header_qry = {"index": ["logs-*"], "ignore_unavailable": True}
search_arr.append(header_qry)
search_arr.append(my_qry)
print('Elastic Query: ' + str(search_arr))
print('------------------------------------------------------------------------------------')
print('Lucene Query: ' + str_query)
request = ''
for each in search_arr:
request += '%s \n' % json.dumps(each)
# print(request)
resp = self.elastic_client.msearch(body=request)
if resp is None and len(resp['responses']) <= 0:
return None
else:
response = resp['responses'][0]
hits_data = list()
if response['hits']['total'] > 0:
for hit in response['hits']['hits']:
hits_data.append(hit)
# print(str(hits_data))
return search_arr, hits_data
def test_invalid_arg(self):
# Test passing not a list
with self.assertRaises(ValueError):
Query.bool(must=set())
# And now an invalid list
with self.assertRaises(ValueError):
Query.bool(must=[None])
# And now an invalid list
with self.assertRaises(ValueError):
Query.bool(must=[Aggregate.terms('test', 'test')])
# And now an invalid list
with self.assertRaises(ValueError):
Query.range('field', gte=['error'])
# Empty list should be OK/ignored
Query.bool(must=[])
def search(self, parms):
device_controller = DeviceController(self.db, self.logger)
if "location" in parms.keys():
list_location_ip = device_controller.get_device_list_by_locationid(
parms["location"])
if "device" in parms.keys():
list_device_ip = device_controller.get_device_list_by_hostname(
parms["device"])
#Doing intersection between device search and device in location
search_ip = list(set(list_location_ip) & set(list_device_ip))
print search_ip
query_search = []
if search_ip:
query_search.append(Query.terms('host', search_ip))
if parms["time"]:
time_from = parms["time"]["from"].split(" ")[0]
time_to = parms["time"]["to"].split(" ")[0]
query_search.append(
Query.range('@timestamp', gte=time_from, lte=time_to))
if parms["severityLevel"]:
query_search.append(
Query.terms('severity', parms["severityLevel"]))
if parms["keywordMessage"]:
message_search = []
message_search.append(parms["keywordMessage"])
query_search.append(Query.terms('message', message_search))
index = "syslog*"
es = Elasticsearch(["http://192.168.100.249:9200"])
q = ElasticQuery(es=es, index=index, doc_type='doc')
# q.query(Query.match_all())
q.size(1000)
q.query(Query.bool(must=query_search))
#q.query(Aggregate.terms(must=query_search))
print q.json(indent=4)
query_result = self.format_results(q.get())
return query_result
#No index to query
else:
return []
'default_operator': 'AND'
}
},
'STRING': {
'query_string': {
'query':
'field_name1:value_name1 AND (field_name2:((value_name2) OR (value_name3)))',
'default_operator': 'AND'
}
},
'NESTED_FILTER': {}
}
# Test filters
print '[ElasticQuery] Testing: filters & queries'
query = Query.range('field_name1', gt=0, lte=100)[1]
test('Query.range', query, FILTERS['RANGE'])
query = Query.prefix(field_name1='value_name1')[1]
test('Query.prefix', query, FILTERS['PREFIX'])
query = Query.term(field_name1='value_name1')[1]
test('Query.term', query, FILTERS['TERM'])
query = Query.terms(field_name1=['value_name1', 'value_name2'])[1]
test('Query.terms', query, FILTERS['TERMS'])
query = Filter.missing('field_name1')[1]
test('Filter.missing', query, FILTERS['FILTER_MISSING'])
query = Query.missing('field_name1')[1]
test('Query.missing', query, FILTERS['QUERY_MISSING'])
