def _on_request_callback(query):
if not tornado_async:
result = es().search(index=index, doc_type=TYPES.get(story_type),
body=query)
return _on_stories_callback(result)
else:
es(tornado_async).search(index=index,
body=json.dumps(query),
doc_type=TYPES.get(story_type),
callback=_on_stories_callback)
def delete_story(owner_id, story_id):
"""Delete a story."""
index = 'app-logs-*'
query = {
'query': {
'bool': {
'filter': {
'bool': {
'must': [
{'term': {'owner_id': owner_id}},
{'term': {'story_id': story_id}},
]
}
}
}
}
}
# Delete all documents matching the above query.
result = es().delete_by_query(index=index, body=query, conflicts='proceed')
if not result['deleted']:
raise NotFoundError('story_id %s' % story_id)
# Report results.
msg = 'Deleted %s log(s) with story_id %s' % (result['deleted'], story_id)
if result['version_conflicts']:
msg += ' Counted %s version_conflicts' % result['version_conflicts']
if result['failures']:
msg += ' Finished with failures: %s' % result['failures']
log.warn(msg)
def _run_query(self, query):
"""Execute the elasticsearch `query` and return the matching documents.
This method must always return the expected result of a query, ex. the
list containing all document hits. Any processing or validation of the
results has to to be done here before returning them.
Subclasses MAY override this method.
"""
return es().search(index=self.index, body=query)['hits']['hits']
def _run_query(self, query):
return es().count(index=self.index, body=query)['count']
def _filtered_query(owner_id, close=None, error=None, range=None, type=None,
callback=None, tornado_async=False, **kwargs):
"""Filter Elasticsearch documents.
Executes a filtering aggregation on Elasticsearch documents in order to
filter by documents indicating a closed story and/or a story that ended
with an error.
The aggregation result constists of two buckets:
- one for closed stories
- one for stories that contain an error
Each bucket sub-aggregates documents based on their `stories` field in
order to group log entries by their associated story IDs.
This method is invoked by mist.api.logs.methods.get_stories.
"""
index = "app-logs-*"
query = {
"query": {
"bool": {
"filter": {
"bool": {
"must": []
}
}
}
},
"size": 0,
"aggs": {
"main_bucket": {
"filters": {
"filters": {}
},
"aggs": {
"stories": {
"terms": {
"field": "stories",
"size": 10000
}
}
}
}
}
}
# Specify whether to filter by closed stories.
if close is not None:
query["aggs"]["main_bucket"]["filters"]["filters"].update(
{"close": {"term": {"stories": "closes"}}}
)
# Specify whether to filter by stories that contain an error.
if error is not None:
query["aggs"]["main_bucket"]["filters"]["filters"].update(
{"error": {"term": {"error": True}}}
)
# Specify the time range of the stories.
if range is not None:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"range": range}
)
# Match the type of the associated stories.
if type:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {"stories": type}}
)
# Fetch logs corresponding to the specified Owner.
if owner_id:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {"owner_id": owner_id}}
)
# Extend query based on additional terms.
for key, value in kwargs.iteritems():
if value in (None, ''):
continue
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {key: value}}
)
# Perform Elasticsearch request.
if not tornado_async:
result = es().search(index=index, doc_type=TYPES.get(type), body=query)
if callback:
return callback(result)
return result
else:
es(tornado_async).search(index=index, doc_type=TYPES.get(type),
body=json.dumps(query), callback=callback)
def get_events(auth_context,
owner_id='',
user_id='',
event_type='',
action='',
limit=0,
start=0,
stop=0,
newest=True,
error=None,
**kwargs):
"""Fetch logged events.
This generator yields a series of logs after querying Elasticsearch.
The initial query is extended with additional terms based on the inputs
provided. Also, extra filtering may be applied in order to perform RBAC
on the fly given the permissions granted to the requesting User.
All Elasticsearch indices are in the form of <app|ui>-logs-<date>.
"""
# Restrict access to UI logs to Admins only.
is_admin = auth_context and auth_context.user.role == 'Admin'
# Attempt to enforce owner_id in case of non-Admins.
if not is_admin and not owner_id:
owner_id = auth_context.owner.id if auth_context else None
# Construct base Elasticsearch query.
index = "%s-logs-*" % ("*" if is_admin else "app")
query = {
"query": {
"bool": {
"filter": {
"bool": {
"must": [{
"range": {
"@timestamp": {
"gte": int(start * 1000),
"lte": int(stop * 1000) or "now"
}
}
}],
"must_not": []
}
}
}
},
"sort": [{
"@timestamp": {
"order": ("desc" if newest else "asc")
}
}],
"size": (limit or 50)
}
# Match action.
if action:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {
'action': action
}})
# Fetch logs corresponding to the current Organization.
if owner_id:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {
"owner_id": owner_id
}})
# Match the user's ID, if provided.
if user_id:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {
"user_id": user_id
}})
# Specify whether to fetch stories that ended with an error.
if error:
query["query"]["bool"]["filter"]["bool"]["must_not"].append(
{"term": {
"error": False
}})
elif error is False:
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {
"error": False
}})
if 'filter' in kwargs:
f = kwargs.pop('filter')
query_string = {
'query': f,
'analyze_wildcard': True,
'default_operator': 'and',
'allow_leading_wildcard': False
}
query["query"]["bool"]["filter"]["bool"]["must"].append(
{'query_string': query_string})
# Extend query with additional kwargs.
for key, value in kwargs.iteritems():
query["query"]["bool"]["filter"]["bool"]["must"].append(
{"term": {
key: value
}})
# Apply RBAC for non-Owners.
if auth_context and not auth_context.is_owner():
filter_logs(auth_context, query)
# Query Elasticsearch.
result = es().search(index=index, doc_type=event_type, body=query)
for hit in result['hits']['hits']:
event = hit['_source']
if not event.get('action'):
log.error('Skipped event %s, missing action', event['log_id'])
continue
try:
extra = json.loads(event.pop('extra'))
except Exception as exc:
log.error('Failed to parse extra of event %s [%s]: '
'%s', event['log_id'], event['action'], exc)
else:
for key, value in extra.iteritems():
event[key] = value
yield event
