def _scroll_query(self, es_url, query): count = None scroll_id = None # Make a "scroll" query to read all answers while count is None or count > 0: data = self._urlopen(es_url, query) yield data if scroll_id is None: if '_scroll_id' not in data: # No data matches the query break scroll_id = data['_scroll_id'] count = data['hits']['total'] es_url = get_es_address() + '_search/scroll' query = bytearray('{"scroll": "1m", "scroll_id": "%s"}' % scroll_id, encoding='utf-8') count -= self.MAX_RESULT_WINDOW # Remove the "scroll" if scroll_id: url = get_es_address() + '_search/scroll' query = '{"scroll_id": "%s"}' % scroll_id self._urlopen(url, query, method='DELETE')
def __init__(self, request, es_address=None, from_date=None, to_date=None, interval=None, qfilter=None): self.from_date = from_date self.to_date = to_date self.interval = interval self.request = request self.qfilter = qfilter if es_address is None: es_address = get_es_address() es_address = es_address.split(',') ssl_params = {} if es_address[0].startswith('https'): ca_certs = None requests_ca_path = os.getenv('REQUESTS_CA_BUNDLE') if requests_ca_path: ca_certs = requests_ca_path ssl_params = { 'use_ssl': True, 'verify_certs': True, 'ca_certs': ca_certs } es = Elasticsearch(es_address, transport_class=ESTransport, **ssl_params) self.es = ESWrap(es)
def _get_es_url(self, data='alert', from_date=None): if from_date is None: from_date = 0 if self.request and 'from_date' in self.request.GET: from_date = self._from_date() if (data == 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX) or (data != 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_INDEX): if data == 'alert': indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX else: indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX else: if from_date == 0: if data == 'alert': indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX + "*" elif data == 'host_id': indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "host_id-*" elif data == 'metricbeat': indexes = 'metricbeat-*' elif data == 'stamus': indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + 'stamus-*' else: indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "*" else: start = datetime.fromtimestamp(int(from_date)/1000) indexes = self._build_es_timestamping(start, data = data) return self.URL % (get_es_address(), indexes)
def get_es_url(from_date, data = 'alert'): if (data == 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX) or (data != 'alert' and '*' in settings.ELASTICSEARCH_LOGSTASH_INDEX): if data == 'alert': indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX else: indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX else: if from_date == 0: if data == 'alert': indexes = settings.ELASTICSEARCH_LOGSTASH_ALERT_INDEX + "*" else: indexes = settings.ELASTICSEARCH_LOGSTASH_INDEX + "*" else: start = datetime.fromtimestamp(int(from_date)/1000) indexes = build_es_timestamping(start, data = data) return URL % (get_es_address(), indexes)
def __init__(self): es_addr = get_es_address() self.client = Elasticsearch([es_addr])