Example #1
0
    def freeSearch(self, searchquery):
        searchquery = re.sub(r'\s+', ' ', searchquery)
        # Elasticsearch connection initialization
        es = Elasticsearch(hosts = [{"host": self.host, "port": self.port}])
        size = 500
        index = ""
        # Find all indexes and remove them from the query
        if searchquery.find("\index") != -1:
            index = searchquery.replace(", ",",").replace(" ",",")[searchquery.find("\index") + 7:]
            searchquery = searchquery[:searchquery.find("\index")]
        q = ElasticQuery(es, index=index, doc_type='')
        ElasticQuery.sort(q,"_score",order="desc")
        ElasticQuery.size(q,size)
        
        # Check correct query syntax (query must have max 2 '\in' and max 1 '\filter')
        if searchquery.count("\in ") <= 2 and searchquery.count("\\filter ") <= 1:
            # Code for query creation like "SELECT *** IN *** FILTER *** IN ***"
            if searchquery.count("\in ") == 2 and searchquery.find("\\filter ") != -1:
                q.query(Query.bool(
                    must=[self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:searchquery.find("\\filter")])],
                    must_not=[self.compose_query(searchquery[searchquery.find("\\filter") + 8:searchquery.rfind("\in")-1],searchquery[searchquery.rfind("\in") + 4:])]
                ))
            
            # Code for query creation like "SELECT *** IN *** FILTER ***"
            elif searchquery.count("\in ") == 1 and searchquery.find("\\filter ") != -1:
                q.query(Query.bool(
                    must=[self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:searchquery.find("\\filter")])],
                    must_not=[self.compose_query(searchquery[searchquery.find("\\filter") + 8:],"_all")]
                ))
            
            # Code for query creation like "SELECT *** IN ***"
            elif searchquery.count("\in ") == 1 and searchquery.find("\\filter ") == -1 and searchquery.find("\\filter") == -1:
                q.query(self.compose_query(searchquery[:searchquery.find("\in")-1],searchquery[searchquery.find("\in") + 4:]))
    
            # Code for query creation like "SELECT ***"
            elif searchquery.count("\in ") == 0 and searchquery.count("\in") == 0 and searchquery.find("\\filter ") == -1 and searchquery.find("\\filter") == -1:
                q.query(self.compose_query(searchquery,"_all"))
            
            # ERROR
            else:
                return HttpResponse('Server: Wrong query syntax!')
        else:
            return HttpResponse('Server: Wrong query syntax!')

        return q.get()
    def query_ec(self, str_query, q_fields, start_date=0, end_date=0, index='logs-*', doc_type='logs',
                 hours=24, debug=False):
        if start_date > end_date:
            raise Exception('The start_date can\'t be greater than the end_date')

        if start_date == 0 or end_date == 0:
            dt_end_date = datetime.now().timestamp()
            dt_start_date = (datetime.now() - timedelta(hours=hours)).timestamp()
            start_date = int(dt_start_date) * 1000
            end_date = int(dt_end_date) * 1000

        # print(str(start_date) + ' -- ' + str(end_date))

        elastic_qry = ElasticQuery(es=self.elastic_client, index=index, doc_type=doc_type)
        elastic_qry.query(
            Query.bool(
                must=[Query.query_string(str_query),
                      Query.range('normalDate', gte=start_date, lte=end_date)]
            )
        )

        elastic_qry.aggregate(
            Aggregate.date_histogram('2', 'normalDate', '12h')
        )

        my_qry = elastic_qry.dict()
        my_qry['stored_fields'] = q_fields

        search_arr = list()
        header_qry = {"index": ["logs-*"], "ignore_unavailable": True}
        search_arr.append(header_qry)
        search_arr.append(my_qry)

        print('Elastic Query: ' + str(search_arr))
        print('------------------------------------------------------------------------------------')
        print('Lucene Query: ' + str_query)

        request = ''
        for each in search_arr:
            request += '%s \n' % json.dumps(each)

        # print(request)

        resp = self.elastic_client.msearch(body=request)

        if resp is None and len(resp['responses']) <= 0:
            return None
        else:
            response = resp['responses'][0]
            hits_data = list()
            if response['hits']['total'] > 0:
                for hit in response['hits']['hits']:
                    hits_data.append(hit)

        # print(str(hits_data))

        return search_arr, hits_data
Example #3
0
    def search(self, parms):
        device_controller = DeviceController(self.db, self.logger)

        if "location" in parms.keys():
            list_location_ip = device_controller.get_device_list_by_locationid(
                parms["location"])

        if "device" in parms.keys():
            list_device_ip = device_controller.get_device_list_by_hostname(
                parms["device"])

        #Doing intersection between device search and device in location
        search_ip = list(set(list_location_ip) & set(list_device_ip))
        print search_ip
        query_search = []
        if search_ip:
            query_search.append(Query.terms('host', search_ip))

            if parms["time"]:
                time_from = parms["time"]["from"].split(" ")[0]
                time_to = parms["time"]["to"].split(" ")[0]
                query_search.append(
                    Query.range('@timestamp', gte=time_from, lte=time_to))

            if parms["severityLevel"]:
                query_search.append(
                    Query.terms('severity', parms["severityLevel"]))

            if parms["keywordMessage"]:
                message_search = []
                message_search.append(parms["keywordMessage"])
                query_search.append(Query.terms('message', message_search))

            index = "syslog*"
            es = Elasticsearch(["http://192.168.100.249:9200"])
            q = ElasticQuery(es=es, index=index, doc_type='doc')

            # q.query(Query.match_all())
            q.size(1000)
            q.query(Query.bool(must=query_search))
            #q.query(Aggregate.terms(must=query_search))

            print q.json(indent=4)
            query_result = self.format_results(q.get())
            return query_result
        #No index to query
        else:
            return []
Example #4
0
    def test_invalid_arg(self):
        # Test passing not a list
        with self.assertRaises(ValueError):
            Query.bool(must=set())

        # And now an invalid list
        with self.assertRaises(ValueError):
            Query.bool(must=[None])

        # And now an invalid list
        with self.assertRaises(ValueError):
            Query.bool(must=[Aggregate.terms('test', 'test')])

        # And now an invalid list
        with self.assertRaises(ValueError):
            Query.range('field', gte=['error'])

        # Empty list should be OK/ignored
        Query.bool(must=[])
Example #5
0
    def logicalSearch(self,query):
        size = 500
        must_fields = ""
        must_values = ""
        should_fields = ""
        should_values = ""
        mustnot_fields = ""
        mustnot_values = ""
        all_indexes = ""

        # Remove space on query string and add % as prefix and suffix searchquery = re.sub(r'\s+', ' ', searchquery)
        query = re.sub(r'\s+', ' ', query).replace(") (",")(").replace("MUST(","%MUST%(").replace("SHOULD(","%SHOULD%(").replace("MUST_NOT(","%MUST_NOT%(").replace("MUST (","%MUST%(").replace("SHOULD (","%SHOULD%(").replace("MUST_NOT (","%MUST_NOT%(") + " %"
        
        # Populate class variables with values only if the relative condition is present on our query
        if query.find("%MUST%") != -1:
            result = self.return_elements(query,"MUST")
            must_fields = self.return_values(result,".","=")
            must_fields = must_fields[:len(self.remove_dupl(must_fields))]
            must_values = self.return_values(result,"=",")")
        if query.find("%SHOULD%") != -1:
            result = self.return_elements(query,"SHOULD")
            should_fields = self.return_values(result,".","=")
            should_fields = should_fields[:len(self.remove_dupl(should_fields))]
            should_values = self.return_values(result,"=",")")
        if query.find("%MUST_NOT%") != -1:
            result = self.return_elements(query,"MUST_NOT")
            mustnot_fields = self.return_values(result,".","=")
            mustnot_fields = mustnot_fields[:len(self.remove_dupl(mustnot_fields))]
            mustnot_values = self.return_values(result,"=",")")
        
        # Elasticsearch connection initialization
        all_indexes = self.return_values(result,"(",".")
        es = Elasticsearch(hosts = [{"host": self.host, "port": self.port}])
        q = ElasticQuery(es,index=self.remove_dupl(all_indexes),doc_type='')
        ElasticQuery.sort(q,"_score",order="desc")
        ElasticQuery.size(q,str(size))

        # Code for query creation like "MUST (...) SHOULD (...) MUST_NOT(...)"
        if must_fields != "" and should_fields != "" and mustnot_fields != "":
            q.query(Query.bool(
                must=[self.compose_query(must_values,must_fields)],
                should=[self.compose_query(should_values,should_fields)],
                must_not=[self.compose_query(mustnot_values,mustnot_fields)]
            ))
        
        # Code for query creation like "MUST (...) SHOULD (...)"
        elif must_fields != "" and should_fields != "" and mustnot_fields == "":
            q.query(Query.bool(
                must=[self.compose_query(must_values,must_fields)],
                should=[self.compose_query(should_values,should_fields)]
            ))
        
        # Code for query creation like "SHOULD (...) MUST_NOT(...)"
        elif must_fields == "" and should_fields != "" and mustnot_fields != "":
            q.query(Query.bool(
                should=[self.compose_query(should_values,should_fields)],
                must_not=[self.compose_query(mustnot_values,mustnot_fields)]
            ))
        
        # Code for query creation like "MUST (...) MUST_NOT(...)"
        elif must_fields != "" and should_fields == "" and mustnot_fields != "":
            q.query(Query.bool(
                must=[self.compose_query(must_values,must_fields)],
                must_not=[self.compose_query(mustnot_values,mustnot_fields)]
            ))
                
        # Code for query creation like "MUST (...)"
        elif must_fields != "" and should_fields == "" and mustnot_fields == "":
            q.query(Query.bool(
                must=[self.compose_query(must_values,must_fields)]
            ))

        # Code for query creation like "SHOULD (...)"
        elif must_fields == "" and should_fields != "" and mustnot_fields == "":
            q.query(Query.bool(
                should=[self.compose_query(should_values,should_fields)]
            ))

        # Code for query creation like "MUST_NOT (...)"
        elif must_fields == "" and should_fields == "" and mustnot_fields != "":
            q.query(Query.bool(
                must_not=[self.compose_query(mustnot_values,mustnot_fields)]
            ))
        
        # ERROR
        else:
            return HttpResponse('Server: Wrong query syntax!')
        
        return q.get()