def with_timer(*args, **kwargs): start = timer() ret = func(*args, **kwargs) end = timer() logger.debug('%s(args=%s, kwargs=%s) takes %s', func.__name__, args, kwargs, end - start) return ret
def with_timer(*args, **kwargs): start = timer() ret = func(*args, **kwargs) end = timer() logger.debug( '%s(args=%s, kwargs=%s) takes %s', func.__name__, args, kwargs, end - start ) return ret
def query_server_error_records(begin_time, end_time): es = Elasticsearch(setting.get('elasticsearch', 'url')) indices = [] day = timedelta(days=1) d = begin_time.date() while d <= end_time.date(): indices.append('logstash-' + d.strftime('%Y.%m.%d')) d += day begin_at = timegm(begin_time.timetuple()) * 1000 end_at = timegm(end_time.timetuple()) * 1000 index = indices = ','.join(indices) offset = 0 results = [] body = { 'filter': { 'bool': { 'must': [ {'range': {'timestamp': {'gte': begin_at, 'lt': end_at}}} ], 'should': [ {'range': {'backend_status_code': {'gte': 500}}}, {'range': {'elb_status_code': {'gte': 500}}} ] } } } while True: body['from'] = offset result = es.search( index=index, body=body, sort='timestamp:asc', size=100, ) logger.debug(result) hits = result.get('hits', {}).get('hits', []) if len(hits) == 0: break results.extend(map(lambda h: h['_source'], hits)) offset += len(hits) return results
def query_server_error_records(begin_time, end_time): es_api = Elasticsearch(setting.get('elasticsearch', 'url')) indices = [ index_name(begin_time.date() + timedelta(n)) for n in xrange((end_time.date() - begin_time.date()).days) ] begin_at = timegm(begin_time.timetuple()) * 1000 end_at = timegm(end_time.timetuple()) * 1000 index = ','.join(indices) if indices else index_name(begin_time.date()) offset = 0 results = [] body = { 'query': { 'bool': { 'filter': [ RangeClause('timestamp', begin_at, end_at).get_clause(), RangeClause('elb_status_code', 500).get_clause(), # Filter out /robots.txt requests ExistClause('rails.controller#action').get_clause(), # Filter out https://52.197.62.134 requests TermClause('domain_name', 'api.thekono.com').get_clause() ] } } } while True: body['from'] = offset args = dict(index=index, body=body, sort='timestamp:asc', size=100) result = es_api.search(**args) logger.debug(result) hits = result.get('hits', {}).get('hits', []) if not hits: break results.extend([h['_source'] for h in hits]) offset += len(hits) return results