def test_misc(self): q = ElasticQuery() q.size(10) q.from_(50) q.timeout('60s') q.set('key', 'value') q.fields(('one_field', 'two_field')) assert_equal(self, q.dict(), { 'size': 10, 'from': 50, 'timeout': '60s', 'key': 'value', '_source': ['one_field', 'two_field'] })
def search(self, parms): device_controller = DeviceController(self.db, self.logger) if "location" in parms.keys(): list_location_ip = device_controller.get_device_list_by_locationid( parms["location"]) if "device" in parms.keys(): list_device_ip = device_controller.get_device_list_by_hostname( parms["device"]) #Doing intersection between device search and device in location search_ip = list(set(list_location_ip) & set(list_device_ip)) print search_ip query_search = [] if search_ip: query_search.append(Query.terms('host', search_ip)) if parms["time"]: time_from = parms["time"]["from"].split(" ")[0] time_to = parms["time"]["to"].split(" ")[0] query_search.append( Query.range('@timestamp', gte=time_from, lte=time_to)) if parms["severityLevel"]: query_search.append( Query.terms('severity', parms["severityLevel"])) if parms["keywordMessage"]: message_search = [] message_search.append(parms["keywordMessage"]) query_search.append(Query.terms('message', message_search)) index = "syslog*" es = Elasticsearch(["http://192.168.100.249:9200"]) q = ElasticQuery(es=es, index=index, doc_type='doc') # q.query(Query.match_all()) q.size(1000) q.query(Query.bool(must=query_search)) #q.query(Aggregate.terms(must=query_search)) print q.json(indent=4) query_result = self.format_results(q.get()) return query_result #No index to query else: return []
def freeSearch(self, searchquery): searchquery = re.sub(r'\s+', ' ', searchquery) # Elasticsearch connection initialization es = Elasticsearch(hosts = [{"host": self.host, "port": self.port}]) size = 500 index = "" # Find all indexes and remove them from the query if searchquery.find("\index") != -1: index = searchquery.replace(", ",",").replace(" ",",")[searchquery.find("\index") + 7:] searchquery = searchquery[:searchquery.find("\index")] q = ElasticQuery(es, index=index, doc_type='') ElasticQuery.sort(q,"_score",order="desc") ElasticQuery.size(q,size) # Check correct query syntax (query must have max 2 '\in' and max 1 '\filter') if searchquery.count("\in ") <= 2 and searchquery.count("\\filter ") <= 1: # Code for query creation like "SELECT *** IN *** FILTER *** IN ***" if searchquery.count("\in ") == 2 and searchquery.find("\\filter ") != -1: q.query(Query.bool( must=[self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:searchquery.find("\\filter")])], must_not=[self.compose_query(searchquery[searchquery.find("\\filter") + 8:searchquery.rfind("\in")-1],searchquery[searchquery.rfind("\in") + 4:])] )) # Code for query creation like "SELECT *** IN *** FILTER ***" elif searchquery.count("\in ") == 1 and searchquery.find("\\filter ") != -1: q.query(Query.bool( must=[self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:searchquery.find("\\filter")])], must_not=[self.compose_query(searchquery[searchquery.find("\\filter") + 8:],"_all")] )) # Code for query creation like "SELECT *** IN ***" elif searchquery.count("\in ") == 1 and searchquery.find("\\filter ") == -1 and searchquery.find("\\filter") == -1: q.query(self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:])) # Code for query creation like "SELECT ***" elif searchquery.count("\in ") == 0 and searchquery.count("\in") == 0 and searchquery.find("\\filter ") == -1 and searchquery.find("\\filter") == -1: q.query(self.compose_query(searchquery,"_all")) # ERROR else: return HttpResponse('Server: Wrong query syntax!') else: return HttpResponse('Server: Wrong query syntax!') return q.get()
def logicalSearch(self,query): size = 500 must_fields = "" must_values = "" should_fields = "" should_values = "" mustnot_fields = "" mustnot_values = "" all_indexes = "" # Remove space on query string and add % as prefix and suffix searchquery = re.sub(r'\s+', ' ', searchquery) query = re.sub(r'\s+', ' ', query).replace(") (",")(").replace("MUST(","%MUST%(").replace("SHOULD(","%SHOULD%(").replace("MUST_NOT(","%MUST_NOT%(").replace("MUST (","%MUST%(").replace("SHOULD (","%SHOULD%(").replace("MUST_NOT (","%MUST_NOT%(") + " %" # Populate class variables with values only if the relative condition is present on our query if query.find("%MUST%") != -1: result = self.return_elements(query,"MUST") must_fields = self.return_values(result,".","=") must_fields = must_fields[:len(self.remove_dupl(must_fields))] must_values = self.return_values(result,"=",")") if query.find("%SHOULD%") != -1: result = self.return_elements(query,"SHOULD") should_fields = self.return_values(result,".","=") should_fields = should_fields[:len(self.remove_dupl(should_fields))] should_values = self.return_values(result,"=",")") if query.find("%MUST_NOT%") != -1: result = self.return_elements(query,"MUST_NOT") mustnot_fields = self.return_values(result,".","=") mustnot_fields = mustnot_fields[:len(self.remove_dupl(mustnot_fields))] mustnot_values = self.return_values(result,"=",")") # Elasticsearch connection initialization all_indexes = self.return_values(result,"(",".") es = Elasticsearch(hosts = [{"host": self.host, "port": self.port}]) q = ElasticQuery(es,index=self.remove_dupl(all_indexes),doc_type='') ElasticQuery.sort(q,"_score",order="desc") ElasticQuery.size(q,str(size)) # Code for query creation like "MUST (...) SHOULD (...) MUST_NOT(...)" if must_fields != "" and should_fields != "" and mustnot_fields != "": q.query(Query.bool( must=[self.compose_query(must_values,must_fields)], should=[self.compose_query(should_values,should_fields)], must_not=[self.compose_query(mustnot_values,mustnot_fields)] )) # Code for query creation like "MUST (...) SHOULD (...)" elif must_fields != "" and should_fields != "" and mustnot_fields == "": q.query(Query.bool( must=[self.compose_query(must_values,must_fields)], should=[self.compose_query(should_values,should_fields)] )) # Code for query creation like "SHOULD (...) MUST_NOT(...)" elif must_fields == "" and should_fields != "" and mustnot_fields != "": q.query(Query.bool( should=[self.compose_query(should_values,should_fields)], must_not=[self.compose_query(mustnot_values,mustnot_fields)] )) # Code for query creation like "MUST (...) MUST_NOT(...)" elif must_fields != "" and should_fields == "" and mustnot_fields != "": q.query(Query.bool( must=[self.compose_query(must_values,must_fields)], must_not=[self.compose_query(mustnot_values,mustnot_fields)] )) # Code for query creation like "MUST (...)" elif must_fields != "" and should_fields == "" and mustnot_fields == "": q.query(Query.bool( must=[self.compose_query(must_values,must_fields)] )) # Code for query creation like "SHOULD (...)" elif must_fields == "" and should_fields != "" and mustnot_fields == "": q.query(Query.bool( should=[self.compose_query(should_values,should_fields)] )) # Code for query creation like "MUST_NOT (...)" elif must_fields == "" and should_fields == "" and mustnot_fields != "": q.query(Query.bool( must_not=[self.compose_query(mustnot_values,mustnot_fields)] )) # ERROR else: return HttpResponse('Server: Wrong query syntax!') return q.get()