def _scroll_query(self, es_url, query):
count = None
scroll_id = None
# Make a "scroll" query to read all answers
while count is None or count > 0:
data = self._urlopen(es_url, query)
yield data
if scroll_id is None:
if '_scroll_id' not in data:
# No data matches the query
break
scroll_id = data['_scroll_id']
count = data['hits']['total']
es_url = get_es_address() + '_search/scroll'
query = bytearray('{"scroll": "1m", "scroll_id": "%s"}' % scroll_id, encoding='utf-8')
count -= self.MAX_RESULT_WINDOW
# Remove the "scroll"
if scroll_id:
url = get_es_address() + '_search/scroll'
query = '{"scroll_id": "%s"}' % scroll_id
self._urlopen(url, query, method='DELETE')
def __init__(self, request, es_address=None, from_date=None, to_date=None, interval=None, qfilter=None):
self.from_date = from_date
self.to_date = to_date
self.interval = interval
self.request = request
self.qfilter = qfilter
if es_address is None:
es_address = get_es_address()
es_address = es_address.split(',')
ssl_params = {}
if es_address[0].startswith('https'):
ca_certs = None
requests_ca_path = os.getenv('REQUESTS_CA_BUNDLE')
if requests_ca_path:
ca_certs = requests_ca_path
ssl_params = {
'use_ssl': True,
'verify_certs': True,
'ca_certs': ca_certs
}
es = Elasticsearch(es_address,
transport_class=ESTransport,
**ssl_params)
self.es = ESWrap(es)
def _get_es_url(self, data='alert', from_date=None):
if from_date is None:
from_date = 0
if self.request and 'from_date' in self.request.GET:
from_date = self._from_date()
if (data == 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX) or (data != 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_INDEX):
if data == 'alert':
indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX
else:
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX
else:
if from_date == 0:
if data == 'alert':
indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX + "*"
elif data == 'host_id':
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "host_id-*"
elif data == 'metricbeat':
indexes = 'metricbeat-*'
elif data == 'stamus':
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + 'stamus-*'
else:
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "*"
else:
start = datetime.fromtimestamp(int(from_date)/1000)
indexes = self._build_es_timestamping(start, data = data)
return self.URL % (get_es_address(), indexes)
def get_es_url(from_date, data = 'alert'):
if (data == 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX) or (data != 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_INDEX):
if data == 'alert':
indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX
else:
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX
else:
if from_date == 0:
if data == 'alert':
indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX + "*"
else:
indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "*"
else:
start = datetime.fromtimestamp(int(from_date)/1000)
indexes = build_es_timestamping(start, data = data)
return URL % (get_es_address(), indexes)
def __init__(self):
es_addr = get_es_address()
self.client = Elasticsearch([es_addr])
