def fetch(self):
logger = logging.getLogger(__name__)
while True:
es = ElasticsearchQuery(es_host=self.ELASTICSEARCH_HOST,
period=self.INTERVAL,
index_prefix=self.ES_INDEX)
res = es.query_by_string(self.QUERY,
fields=self.FIELDS,
limit=self.BATCH)
urls = map(self.format_log_entry, res)
for url in urls:
if self.filter_out(url):
logger.info('Filtered out <%s>', url)
continue
yield url
time.sleep(self.INTERVAL)
def get_log_entries(query,
period,
fields,
limit,
index_prefix='logstash-other'):
"""
Get log entries from elasticsearch that match given query
:type query str
:type period int
:type fields list[str] or None
:type limit int
:type index_prefix str
:rtype tuple
"""
logger = logging.getLogger('get_log_entries')
source = ElasticsearchQuery(es_host=LOGS_ES_HOST,
period=period,
index_prefix=index_prefix)
logger.info('Query: \'%s\' for the last %d hour(s)', query, period / 3600)
return source.query_by_string(query, fields, limit)