def with_timer(*args, **kwargs):
start = timer()
ret = func(*args, **kwargs)
end = timer()
logger.debug('%s(args=%s, kwargs=%s) takes %s', func.__name__, args,
kwargs, end - start)
return ret
def with_timer(*args, **kwargs):
start = timer()
ret = func(*args, **kwargs)
end = timer()
logger.debug(
'%s(args=%s, kwargs=%s) takes %s',
func.__name__,
args,
kwargs,
end - start
)
return ret
def query_server_error_records(begin_time, end_time):
es = Elasticsearch(setting.get('elasticsearch', 'url'))
indices = []
day = timedelta(days=1)
d = begin_time.date()
while d <= end_time.date():
indices.append('logstash-' + d.strftime('%Y.%m.%d'))
d += day
begin_at = timegm(begin_time.timetuple()) * 1000
end_at = timegm(end_time.timetuple()) * 1000
index = indices = ','.join(indices)
offset = 0
results = []
body = {
'filter': {
'bool': {
'must': [
{'range': {'timestamp': {'gte': begin_at, 'lt': end_at}}}
],
'should': [
{'range': {'backend_status_code': {'gte': 500}}},
{'range': {'elb_status_code': {'gte': 500}}}
]
}
}
}
while True:
body['from'] = offset
result = es.search(
index=index,
body=body,
sort='timestamp:asc',
size=100,
)
logger.debug(result)
hits = result.get('hits', {}).get('hits', [])
if len(hits) == 0:
break
results.extend(map(lambda h: h['_source'], hits))
offset += len(hits)
return results
def query_server_error_records(begin_time, end_time):
es_api = Elasticsearch(setting.get('elasticsearch', 'url'))
indices = [
index_name(begin_time.date() + timedelta(n))
for n in xrange((end_time.date() - begin_time.date()).days)
]
begin_at = timegm(begin_time.timetuple()) * 1000
end_at = timegm(end_time.timetuple()) * 1000
index = ','.join(indices) if indices else index_name(begin_time.date())
offset = 0
results = []
body = {
'query': {
'bool': {
'filter': [
RangeClause('timestamp', begin_at, end_at).get_clause(),
RangeClause('elb_status_code', 500).get_clause(),
# Filter out /robots.txt requests
ExistClause('rails.controller#action').get_clause(),
# Filter out https://52.197.62.134 requests
TermClause('domain_name', 'api.thekono.com').get_clause()
]
}
}
}
while True:
body['from'] = offset
args = dict(index=index, body=body, sort='timestamp:asc', size=100)
result = es_api.search(**args)
logger.debug(result)
hits = result.get('hits', {}).get('hits', [])
if not hits:
break
results.extend([h['_source'] for h in hits])
offset += len(hits)
return results
