def add_view(self, name, query_string=None, query_dsl=None,
query_filter=None):
"""Add saved view to the Sketch.
Args:
name: The name of the view.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not query_string or query_dsl:
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
view = View(name=name, sketch=self.sql_sketch, user=None,
query_string=query_string, query_filter=query_filter,
query_dsl=query_dsl, searchtemplate=None)
view.query_filter = view.validate_filter(query_filter)
db_session.add(view)
db_session.commit()
return view
def add_view(self, view_name, analyzer_name, query_string=None,
query_dsl=None, query_filter=None):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not query_string or query_dsl:
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
name = '[{0:s}] {1:s}'.format(analyzer_name, view_name)
view = View.get_or_create(name=name, sketch=self.sql_sketch, user=None)
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
db_session.add(view)
db_session.commit()
return view
def update_sketch_last_activity(sketch):
"""Update the last activity date of a sketch."""
view = View.get_or_create(user=current_user, sketch=sketch, name='')
view.update_modification_time()
db_session.add(view)
db_session.commit()
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
if view_id:
view = View.query.get(view_id)
else:
view = View.query.filter(View.user == current_user, View.name == u'',
View.sketch_id == sketch_id).order_by(
View.created_at.desc()).first()
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(user=current_user,
name=u'',
sketch=sketch,
query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
view_form = SaveViewForm()
return render_template(u'sketch/explore.html',
sketch=sketch,
view=view,
timelines=sketch_timelines,
view_form=view_form)
def add_view(self, view_name, analyzer_name, query_string=None,
query_dsl=None, query_filter=None):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not query_string or query_dsl:
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
name = '[{0:s}] {1:s}'.format(analyzer_name, view_name)
view = View.get_or_create(name=name, sketch=self.sql_sketch, user=None)
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
db_session.add(view)
db_session.commit()
return view
def add_view(
self,
view_name,
analyzer_name,
query_string=None,
query_dsl=None,
query_filter=None,
additional_fields=None,
):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: OpenSearch query string.
query_dsl: Dictionary with OpenSearch DSL query.
query_filter: Dictionary with OpenSearch filters.
additional_fields: A list with field names to include in the
view output.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not (query_string or query_dsl):
raise ValueError("Both query_string and query_dsl are missing.")
if not query_filter:
query_filter = {"indices": "_all"}
if additional_fields:
query_filter["fields"] = [{
"field": x.strip()
} for x in additional_fields]
description = "analyzer: {0:s}".format(analyzer_name)
view = View.get_or_create(name=view_name,
description=description,
sketch=self.sql_sketch,
user=None)
view.description = description
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
view.set_status(status="new")
db_session.add(view)
db_session.commit()
return view
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
if view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(user=current_user,
name=u'',
sketch=sketch,
query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return render_template(u'sketch/explore.html',
sketch=sketch,
view=view,
timelines=sketch_timelines,
view_form=view_form)
def _create_view(self, name, sketch, user):
"""Create a view in the database.
Args:
name: Name of the view (string)
sketch: A sketch (instance of timesketch.models.sketch.Sketch)
user: A user (instance of timesketch.models.user.User)
Returns:
A view (instance of timesketch.models.sketch.View)
"""
view = View(name=name,
query_string=name,
query_filter=json.dumps(dict()),
user=user,
sketch=sketch)
self._commit_to_database(view)
return view
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = SaveViewForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
view = View(
name=form.name.data, sketch=sketch, user=current_user,
query_string=form.query.data,
query_filter=json.dumps(form.filter.data, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
form = ExploreForm.build(request)
if form.validate_on_submit():
query_filter = form.filter.data
sketch_indices = [
t.searchindex.index_name for t in sketch.timelines]
indices = query_filter.get(u'indices', sketch_indices)
# Make sure that the indices in the filter are part of the sketch
if set(indices) - set(sketch_indices):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Make sure we have a query string or star filter
if not form.query.data and not query_filter.get(u'star'):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
result = self.datastore.search(
sketch_id, form.query.data, query_filter, indices)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result[u'hits'][u'hits']:
event[u'selected'] = False
event[u'_source'][u'label'] = []
try:
for label in event[u'_source'][u'timesketch_label']:
if sketch.id != label[u'sketch_id']:
continue
event[u'_source'][u'label'].append(label[u'name'])
del event[u'_source'][u'timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
view = View.get_or_create(
user=current_user, sketch=sketch, name=u'')
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
u'es_time': result[u'took'],
u'es_total_count': result[u'hits'][u'total'],
u'timeline_colors': tl_colors,
u'timeline_names': tl_names
}
schema = {
u'meta': meta,
u'objects': result[u'hits'][u'hits']
}
return jsonify(schema)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def explore(sketch_id, view_id=None, searchtemplate_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI')
# Get parameters from the GET query
url_query = request.args.get('q', '')
url_time_start = request.args.get('time_start', None)
url_time_end = request.args.get('time_end', None)
url_index = request.args.get('index', None)
url_size = request.args.get('size', None)
if searchtemplate_id:
searchtemplate = SearchTemplate.query.get(searchtemplate_id)
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user, name='', sketch=sketch)
view.query_string = searchtemplate.query_string
view.query_filter = searchtemplate.query_filter
view.query_dsl = searchtemplate.query_dsl
save_view = True
elif view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == 'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user,
name='',
sketch=sketch,
query_string='*')
view.query_filter = view.validate_filter(
dict(indices=sketch_timelines))
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter['from'] = 0 # if we loaded from get, start at first event
query_filter['time_start'] = url_time_start
query_filter['time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter['indices'] = [url_index]
if url_size:
query_filter['size'] = url_size
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = None
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template('sketch/explore.html',
sketch=sketch,
view=view,
named_view=view_id,
timelines=sketch_timelines,
view_form=view_form,
searchtemplate_id=searchtemplate_id,
graphs_enabled=graphs_enabled,
similarity_enabled=similarity_enabled)
def explore(sketch_id, view_id=None, searchtemplate_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI')
# Get parameters from the GET query
url_query = request.args.get('q', '')
url_time_start = request.args.get('time_start', None)
url_time_end = request.args.get('time_end', None)
url_index = request.args.get('index', None)
url_size = request.args.get('size', None)
if searchtemplate_id:
searchtemplate = SearchTemplate.query.get(searchtemplate_id)
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user, name='', sketch=sketch)
view.query_string = searchtemplate.query_string
view.query_filter = searchtemplate.query_filter
view.query_dsl = searchtemplate.query_dsl
save_view = True
elif view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == 'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
view = View(
user=current_user, name='', sketch=sketch, query_string='*')
view.query_filter = view.validate_filter(
dict(indices=sketch_timelines))
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter['from'] = 0 # if we loaded from get, start at first event
query_filter['time_start'] = url_time_start
query_filter['time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter['indices'] = [url_index]
if url_size:
query_filter['size'] = url_size
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = None
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template(
'sketch/explore.html',
sketch=sketch,
view=view,
named_view=view_id,
timelines=sketch_timelines,
view_form=view_form,
searchtemplate_id=searchtemplate_id,
graphs_enabled=graphs_enabled,
similarity_enabled=similarity_enabled)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
if not sketch.has_permission(current_user, "read"):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
"User does not have read access controls on sketch.",
)
if sketch.get_status.status == "archived":
abort(HTTP_STATUS_CODE_BAD_REQUEST,
"Unable to query on an archived sketch.")
form = forms.ExploreForm.build(request)
if not form.validate_on_submit():
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"Unable to explore data, unable to validate form data",
)
# TODO: Remove form and use json instead.
query_dsl = form.dsl.data
enable_scroll = form.enable_scroll.data
scroll_id = form.scroll_id.data
file_name = form.file_name.data
count = bool(form.count.data)
query_filter = request.json.get("filter", {})
parent = request.json.get("parent", None)
incognito = request.json.get("incognito", False)
return_field_string = form.fields.data
if return_field_string:
return_fields = [x.strip() for x in return_field_string.split(",")]
else:
return_fields = query_filter.get("fields", [])
return_fields = [field["field"] for field in return_fields]
return_fields.extend(DEFAULT_SOURCE_FIELDS)
if not query_filter:
query_filter = {}
all_indices = list(
{t.searchindex.index_name
for t in sketch.timelines})
indices = query_filter.get("indices", all_indices)
# If _all in indices then execute the query on all indices
if "_all" in indices:
indices = all_indices
# Make sure that the indices in the filter are part of the sketch.
# This will also remove any deleted timeline from the search result.
indices, timeline_ids = get_validated_indices(indices, sketch)
# Remove indices that don't exist from search.
indices = utils.validate_indices(indices, self.datastore)
if not indices:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"No valid search indices were found to perform the search on.",
)
# Make sure we have a query string or star filter
if not (
form.query.data,
query_filter.get("star"),
query_filter.get("events"),
query_dsl,
):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"The request needs a query string/DSL and or a star filter.",
)
# Aggregate hit count per index.
index_stats_agg = {
"indices": {
"terms": {
"field": "_index",
"min_doc_count": 0,
"size": len(sketch.timelines),
}
},
"timelines": {
"terms": {
"field": "__ts_timeline_id",
"min_doc_count": 0,
"size": len(sketch.timelines),
}
},
"count_over_time": {
"auto_date_histogram": {
"field": "datetime",
"buckets": 50,
}
},
}
if count:
# Count operations do not support size parameters.
if "size" in query_filter:
_ = query_filter.pop("size")
if "terminate_after" in query_filter:
_ = query_filter.pop("terminate_after")
try:
result = self.datastore.search(
sketch_id=sketch_id,
query_string=form.query.data,
query_filter=query_filter,
query_dsl=query_dsl,
indices=indices,
timeline_ids=timeline_ids,
count=True,
)
except ValueError as e:
abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))
# Get number of matching documents per index.
schema = {"meta": {"total_count": result}, "objects": []}
return jsonify(schema)
if file_name:
file_object = io.BytesIO()
form_data = {
"created_at": datetime.datetime.utcnow().isoformat(),
"created_by": current_user.username,
"sketch": sketch_id,
"query": form.query.data,
"query_dsl": query_dsl,
"query_filter": query_filter,
"return_fields": return_fields,
}
with zipfile.ZipFile(file_object, mode="w") as zip_file:
zip_file.writestr("METADATA", data=json.dumps(form_data))
fh = export.query_to_filehandle(
query_string=form.query.data,
query_dsl=query_dsl,
query_filter=query_filter,
indices=indices,
sketch=sketch,
datastore=self.datastore,
return_fields=return_fields,
timeline_ids=timeline_ids,
)
fh.seek(0)
zip_file.writestr("query_results.csv", fh.read())
file_object.seek(0)
return send_file(file_object,
mimetype="zip",
attachment_filename=file_name)
if scroll_id:
# pylint: disable=unexpected-keyword-arg
result = self.datastore.client.scroll(scroll_id=scroll_id,
scroll="1m")
else:
try:
result = self.datastore.search(
sketch_id=sketch_id,
query_string=form.query.data,
query_filter=query_filter,
query_dsl=query_dsl,
indices=indices,
aggregations=index_stats_agg,
return_fields=return_fields,
enable_scroll=enable_scroll,
timeline_ids=timeline_ids,
)
except ValueError as e:
abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))
# Get number of matching documents over time.
histogram_interval = (result.get("aggregations",
{}).get("count_over_time",
{}).get("interval", ""))
count_over_time = {"data": {}, "interval": histogram_interval}
try:
for bucket in result["aggregations"]["count_over_time"]["buckets"]:
key = bucket.get("key")
if key:
count_over_time["data"][key] = bucket.get("doc_count")
except KeyError:
pass
# Get number of matching documents per index.
count_per_index = {}
try:
for bucket in result["aggregations"]["indices"]["buckets"]:
key = bucket.get("key")
if key:
count_per_index[key] = bucket.get("doc_count")
except KeyError:
pass
# Get number of matching documents per timeline.
count_per_timeline = {}
try:
for bucket in result["aggregations"]["timelines"]["buckets"]:
key = bucket.get("key")
if key:
count_per_timeline[key] = bucket.get("doc_count")
except KeyError:
pass
# Total count for query regardless of returned results.
count_total_complete = sum(count_per_index.values())
comments = {}
if "comment" in return_fields:
events = Event.query.filter_by(sketch=sketch).all()
for event in events:
for comment in event.comments:
comments.setdefault(event.document_id, [])
comments[event.document_id].append(comment.comment)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result["hits"]["hits"]:
event["selected"] = False
event["_source"]["label"] = []
try:
for label in event["_source"]["timesketch_label"]:
if sketch.id != label["sketch_id"]:
continue
event["_source"]["label"].append(label["name"])
del event["_source"]["timesketch_label"]
except KeyError:
pass
if "comment" in return_fields:
event["_source"]["comment"] = comments.get(event["_id"], [])
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
# TODO: Deprecate this and change how last activity is determined, e.g
# use the new Search History feature instead.
view = View.get_or_create(user=current_user, sketch=sketch, name="")
view.update_modification_time()
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Search History
search_node = None
new_search = SearchHistory(user=current_user, sketch=sketch)
if parent:
previous_search = SearchHistory.query.get(parent)
else:
previous_search = (SearchHistory.query.filter_by(
user=current_user,
sketch=sketch).order_by(SearchHistory.id.desc()).first())
if not incognito:
is_same_query = False
is_same_filter = False
new_search.query_string = form.query.data
new_search.query_filter = json.dumps(query_filter,
ensure_ascii=False)
new_search.query_result_count = count_total_complete
new_search.query_time = result["took"]
if previous_search:
new_search.parent = previous_search
new_query = new_search.query_string
new_filter = new_search.query_filter
previous_query = previous_search.query_string
previous_filter = previous_search.query_filter
is_same_query = previous_query == new_query
is_same_filter = previous_filter == new_filter
if not all([is_same_query, is_same_filter]):
db_session.add(new_search)
db_session.commit()
# Create metric if user creates a new branch.
if new_search.parent:
if len(new_search.parent.children) > 1:
METRICS["searchhistory"].labels(action="branch").inc()
else:
METRICS["searchhistory"].labels(
action="ignore_same_query").inc()
else:
METRICS["searchhistory"].labels(action="incognito").inc()
search_node = new_search if new_search.id else previous_search
if not search_node:
abort(HTTP_STATUS_CODE_BAD_REQUEST, "Unable to save search")
search_node = search_node.build_tree(search_node, {}, recurse=False)
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
"es_time": result["took"],
"es_total_count": result["hits"]["total"],
"es_total_count_complete": count_total_complete,
"timeline_colors": tl_colors,
"timeline_names": tl_names,
"count_per_index": count_per_index,
"count_per_timeline": count_per_timeline,
"count_over_time": count_over_time,
"scroll_id": result.get("_scroll_id", ""),
"search_node": search_node,
}
# Elasticsearch version 7.x returns total hits as a dictionary.
# TODO: Refactor when version 6.x has been deprecated.
if isinstance(meta["es_total_count"], dict):
meta["es_total_count"] = meta["es_total_count"].get("value", 0)
schema = {"meta": meta, "objects": result["hits"]["hits"]}
return jsonify(schema)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
form = ExploreForm.build(request)
if form.validate_on_submit():
query_dsl = form.dsl.data
query_filter = form.filter.data
sketch_indices = {
t.searchindex.index_name
for t in sketch.timelines
}
indices = query_filter.get(u'indices', sketch_indices)
# If _all in indices then execute the query on all indices
if u'_all' in indices:
indices = sketch_indices
# Make sure that the indices in the filter are part of the sketch
if set(indices) - set(sketch_indices):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Make sure we have a query string or star filter
if not (form.query.data, query_filter.get(u'star'),
query_filter.get(u'events'), query_dsl):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
result = self.datastore.search(sketch_id,
form.query.data,
query_filter,
query_dsl,
indices,
aggregations=None,
return_results=True)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result[u'hits'][u'hits']:
event[u'selected'] = False
event[u'_source'][u'label'] = []
try:
for label in event[u'_source'][u'timesketch_label']:
if sketch.id != label[u'sketch_id']:
continue
event[u'_source'][u'label'].append(label[u'name'])
del event[u'_source'][u'timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
view = View.get_or_create(user=current_user,
sketch=sketch,
name=u'')
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
try:
buckets = result[u'aggregations'][u'field_aggregation'][
u'buckets']
except KeyError:
buckets = None
es_total_count_unfiltered = 0
if buckets:
for bucket in buckets:
es_total_count_unfiltered += bucket[u'doc_count']
meta = {
u'es_time': result[u'took'],
u'es_total_count': result[u'hits'][u'total'],
u'es_total_count_unfiltered': es_total_count_unfiltered,
u'timeline_colors': tl_colors,
u'timeline_names': tl_names,
u'histogram': buckets
}
schema = {u'meta': meta, u'objects': result[u'hits'][u'hits']}
return jsonify(schema)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def explore(sketch_id, view_id=None, searchtemplate_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
url_index = request.args.get(u'index', None)
url_limit = request.args.get(u'limit', None)
if searchtemplate_id:
searchtemplate = SearchTemplate.query.get(searchtemplate_id)
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user, name=u'', sketch=sketch)
view.query_string = searchtemplate.query_string
view.query_filter = searchtemplate.query_filter
view.query_dsl = searchtemplate.query_dsl
save_view = True
elif view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'*')
view.query_filter = view.validate_filter(
dict(indices=sketch_timelines))
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter[u'indices'] = [url_index]
if url_limit:
query_filter[u'limit'] = url_limit
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = None
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view, named_view=view_id,
timelines=sketch_timelines, view_form=view_form,
searchtemplate_id=searchtemplate_id)
def create_view_from_form(sketch, form):
"""Creates a view from form data.
Args:
sketch: Instance of timesketch.models.sketch.Sketch
form: Instance of timesketch.lib.forms.SaveViewForm
Returns:
A view (Instance of timesketch.models.sketch.View)
"""
# Default to user supplied data
view_name = form.name.data
query_string = form.query.data
query_filter_dict = form.filter.data
# Stripping potential pagination from views before saving it.
if 'from' in query_filter_dict:
del query_filter_dict['from']
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
query_dsl = json.dumps(form.dsl.data, ensure_ascii=False)
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# No search template by default (before we know if the user want to
# create a template or use an existing template when creating the view)
searchtemplate = None
# Create view from a search template
if form.from_searchtemplate_id.data:
# Get the template from the datastore
template_id = form.from_searchtemplate_id.data
searchtemplate = SearchTemplate.query.get(template_id)
# Copy values from the template
view_name = searchtemplate.name
query_string = searchtemplate.query_string
query_filter = searchtemplate.query_filter
query_dsl = searchtemplate.query_dsl
# WTF form returns a tuple for the filter. This is not
# compatible with SQLAlchemy.
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
if not view_name:
abort(HTTP_STATUS_CODE_BAD_REQUEST, 'View name is missing.')
# Create a new search template based on this view (only if requested by
# the user).
if form.new_searchtemplate.data:
query_filter_dict = json.loads(query_filter)
if query_filter_dict.get('indices', None):
query_filter_dict['indices'] = '_all'
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
searchtemplate = SearchTemplate(name=view_name,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl)
db_session.add(searchtemplate)
db_session.commit()
# Create the view in the database
view = View(name=view_name,
sketch=sketch,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl,
searchtemplate=searchtemplate)
db_session.add(view)
db_session.commit()
return view
def create_view_from_form(sketch, form):
"""Creates a view from form data.
Args:
sketch: Instance of timesketch.models.sketch.Sketch
form: Instance of timesketch.lib.forms.SaveViewForm
Returns:
A view (Instance of timesketch.models.sketch.View)
"""
# Default to user supplied data
view_name = form.name.data
query_string = form.query.data
query_filter = json.dumps(form.filter.data, ensure_ascii=False),
query_dsl = json.dumps(form.dsl.data, ensure_ascii=False)
# WTF forms turns the filter into a tuple for some reason.
# pylint: disable=redefined-variable-type
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# No search template by default (before we know if the user want to
# create a template or use an existing template when creating the view)
searchtemplate = None
# Create view from a search template
if form.from_searchtemplate_id.data:
# Get the template from the datastore
template_id = form.from_searchtemplate_id.data
searchtemplate = SearchTemplate.query.get(template_id)
# Copy values from the template
view_name = searchtemplate.name
query_string = searchtemplate.query_string
query_filter = searchtemplate.query_filter,
query_dsl = searchtemplate.query_dsl
# WTF form returns a tuple for the filter. This is not
# compatible with SQLAlchemy.
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# Create a new search template based on this view (only if requested by
# the user).
if form.new_searchtemplate.data:
query_filter_dict = json.loads(query_filter)
if query_filter_dict.get(u'indices', None):
query_filter_dict[u'indices'] = u'_all'
# pylint: disable=redefined-variable-type
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
searchtemplate = SearchTemplate(name=view_name,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl)
db_session.add(searchtemplate)
db_session.commit()
# Create the view in the database
view = View(name=view_name,
sketch=sketch,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl,
searchtemplate=searchtemplate)
db_session.add(view)
db_session.commit()
return view
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'read'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have read access controls on sketch.')
if sketch.get_status.status == 'archived':
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to query on an archived sketch.')
form = forms.ExploreForm.build(request)
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to explore data, unable to validate form data')
# TODO: Remove form and use json instead.
query_dsl = form.dsl.data
enable_scroll = form.enable_scroll.data
scroll_id = form.scroll_id.data
file_name = form.file_name.data
count = bool(form.count.data)
query_filter = request.json.get('filter', {})
parent = request.json.get('parent', None)
incognito = request.json.get('incognito', False)
return_field_string = form.fields.data
if return_field_string:
return_fields = [x.strip() for x in return_field_string.split(',')]
else:
return_fields = query_filter.get('fields', [])
return_fields = [field['field'] for field in return_fields]
return_fields.extend(DEFAULT_SOURCE_FIELDS)
if not query_filter:
query_filter = {}
all_indices = list(
{t.searchindex.index_name
for t in sketch.timelines})
indices = query_filter.get('indices', all_indices)
# If _all in indices then execute the query on all indices
if '_all' in indices:
indices = all_indices
# Make sure that the indices in the filter are part of the sketch.
# This will also remove any deleted timeline from the search result.
indices, timeline_ids = get_validated_indices(indices, sketch)
# Remove indices that don't exist from search.
indices = utils.validate_indices(indices, self.datastore)
if not indices:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'No valid search indices were found to perform the search on.')
# Make sure we have a query string or star filter
if not (form.query.data, query_filter.get('star'),
query_filter.get('events'), query_dsl):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'The request needs a query string/DSL and or a star filter.')
# Aggregate hit count per index.
index_stats_agg = {
'indices': {
'terms': {
'field': '_index',
'min_doc_count': 0,
'size': len(sketch.timelines)
}
},
'timelines': {
'terms': {
'field': '__ts_timeline_id',
'min_doc_count': 0,
'size': len(sketch.timelines)
}
}
}
if count:
# Count operations do not support size parameters.
if 'size' in query_filter:
_ = query_filter.pop('size')
if 'terminate_after' in query_filter:
_ = query_filter.pop('terminate_after')
try:
result = self.datastore.search(sketch_id=sketch_id,
query_string=form.query.data,
query_filter=query_filter,
query_dsl=query_dsl,
indices=indices,
timeline_ids=timeline_ids,
count=True)
except ValueError as e:
abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))
# Get number of matching documents per index.
schema = {'meta': {'total_count': result}, 'objects': []}
return jsonify(schema)
if file_name:
file_object = io.BytesIO()
form_data = {
'created_at': datetime.datetime.utcnow().isoformat(),
'created_by': current_user.username,
'sketch': sketch_id,
'query': form.query.data,
'query_dsl': query_dsl,
'query_filter': query_filter,
'return_fields': return_fields,
}
with zipfile.ZipFile(file_object, mode='w') as zip_file:
zip_file.writestr('METADATA', data=json.dumps(form_data))
fh = export.query_to_filehandle(query_string=form.query.data,
query_dsl=query_dsl,
query_filter=query_filter,
indices=indices,
sketch=sketch,
datastore=self.datastore,
return_fields=return_fields,
timeline_ids=timeline_ids)
fh.seek(0)
zip_file.writestr('query_results.csv', fh.read())
file_object.seek(0)
return send_file(file_object,
mimetype='zip',
attachment_filename=file_name)
if scroll_id:
# pylint: disable=unexpected-keyword-arg
result = self.datastore.client.scroll(scroll_id=scroll_id,
scroll='1m')
else:
try:
result = self.datastore.search(sketch_id=sketch_id,
query_string=form.query.data,
query_filter=query_filter,
query_dsl=query_dsl,
indices=indices,
aggregations=index_stats_agg,
return_fields=return_fields,
enable_scroll=enable_scroll,
timeline_ids=timeline_ids)
except ValueError as e:
abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))
# Get number of matching documents per index.
count_per_index = {}
try:
for bucket in result['aggregations']['indices']['buckets']:
key = bucket.get('key')
if key:
count_per_index[key] = bucket.get('doc_count')
except KeyError:
pass
# Get number of matching documents per timeline.
count_per_timeline = {}
try:
for bucket in result['aggregations']['timelines']['buckets']:
key = bucket.get('key')
if key:
count_per_timeline[key] = bucket.get('doc_count')
except KeyError:
pass
# Total count for query regardless of returned results.
count_total_complete = sum(count_per_index.values())
comments = {}
if 'comment' in return_fields:
events = Event.query.filter_by(sketch=sketch).all()
for event in events:
for comment in event.comments:
comments.setdefault(event.document_id, [])
comments[event.document_id].append(comment.comment)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result['hits']['hits']:
event['selected'] = False
event['_source']['label'] = []
try:
for label in event['_source']['timesketch_label']:
if sketch.id != label['sketch_id']:
continue
event['_source']['label'].append(label['name'])
del event['_source']['timesketch_label']
except KeyError:
pass
if 'comment' in return_fields:
event['_source']['comment'] = comments.get(event['_id'], [])
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
# TODO: Deprecate this and change how last activity is determined, e.g
# use the new Search History feature instead.
view = View.get_or_create(user=current_user, sketch=sketch, name='')
view.update_modification_time()
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Search History
search_node = None
new_search = SearchHistory(user=current_user, sketch=sketch)
if parent:
previous_search = SearchHistory.query.get(parent)
else:
previous_search = SearchHistory.query.filter_by(
user=current_user,
sketch=sketch).order_by(SearchHistory.id.desc()).first()
if not incognito:
is_same_query = False
is_same_filter = False
new_search.query_string = form.query.data
new_search.query_filter = json.dumps(query_filter,
ensure_ascii=False)
new_search.query_result_count = count_total_complete
new_search.query_time = result['took']
if previous_search:
new_search.parent = previous_search
new_query = new_search.query_string
new_filter = new_search.query_filter
previous_query = previous_search.query_string
previous_filter = previous_search.query_filter
is_same_query = previous_query == new_query
is_same_filter = previous_filter == new_filter
if not all([is_same_query, is_same_filter]):
db_session.add(new_search)
db_session.commit()
# Create metric if user creates a new branch.
if new_search.parent:
if len(new_search.parent.children) > 1:
METRICS['searchhistory'].labels(action='branch').inc()
else:
METRICS['searchhistory'].labels(
action='ignore_same_query').inc()
else:
METRICS['searchhistory'].labels(action='incognito').inc()
search_node = new_search if new_search.id else previous_search
if not search_node:
abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to save search')
search_node = search_node.build_tree(search_node, {}, recurse=False)
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
'es_time': result['took'],
'es_total_count': result['hits']['total'],
'es_total_count_complete': count_total_complete,
'timeline_colors': tl_colors,
'timeline_names': tl_names,
'count_per_index': count_per_index,
'count_per_timeline': count_per_timeline,
'scroll_id': result.get('_scroll_id', ''),
'search_node': search_node
}
# Elasticsearch version 7.x returns total hits as a dictionary.
# TODO: Refactor when version 6.x has been deprecated.
if isinstance(meta['es_total_count'], dict):
meta['es_total_count'] = meta['es_total_count'].get('value', 0)
schema = {'meta': meta, 'objects': result['hits']['hits']}
return jsonify(schema)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'read'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have read access controls on sketch.')
if sketch.get_status.status == 'archived':
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to query on an archived sketch.')
form = forms.ExploreForm.build(request)
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to explore data, unable to validate form data')
# TODO: Remove form and use json instead.
query_dsl = form.dsl.data
enable_scroll = form.enable_scroll.data
scroll_id = form.scroll_id.data
file_name = form.file_name.data
query_filter = request.json.get('filter', {})
return_field_string = form.fields.data
if return_field_string:
return_fields = [x.strip() for x in return_field_string.split(',')]
else:
return_fields = query_filter.get('fields', [])
return_fields = [field['field'] for field in return_fields]
return_fields.extend(DEFAULT_SOURCE_FIELDS)
sketch_indices = {
t.searchindex.index_name
for t in sketch.timelines if t.get_status.status.lower() == 'ready'
}
if not query_filter:
query_filter = {}
indices = query_filter.get('indices', sketch_indices)
# If _all in indices then execute the query on all indices
if '_all' in indices:
indices = sketch_indices
# Make sure that the indices in the filter are part of the sketch.
# This will also remove any deleted timeline from the search result.
indices = get_validated_indices(indices, sketch_indices)
# Make sure we have a query string or star filter
if not (form.query.data, query_filter.get('star'),
query_filter.get('events'), query_dsl):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'The request needs a query string/DSL and or a star filter.')
# Aggregate hit count per index.
index_stats_agg = {"indices": {"terms": {"field": "_index"}}}
if file_name:
file_object = io.BytesIO()
form_data = {
'created_at': datetime.datetime.utcnow().isoformat(),
'created_by': current_user.username,
'sketch': sketch_id,
'query': form.query.data,
'query_dsl': query_dsl,
'query_filter': query_filter,
'return_fields': return_fields,
}
with zipfile.ZipFile(file_object, mode='w') as zip_file:
zip_file.writestr('METADATA', data=json.dumps(form_data))
fh = export.query_to_filehandle(query_string=form.query.data,
query_dsl=query_dsl,
query_filter=query_filter,
indices=indices,
sketch=sketch,
datastore=self.datastore)
fh.seek(0)
zip_file.writestr('query_results.csv', fh.read())
file_object.seek(0)
return send_file(file_object,
mimetype='zip',
attachment_filename=file_name)
if scroll_id:
# pylint: disable=unexpected-keyword-arg
result = self.datastore.client.scroll(scroll_id=scroll_id,
scroll='1m')
else:
try:
result = self.datastore.search(sketch_id,
form.query.data,
query_filter,
query_dsl,
indices,
aggregations=index_stats_agg,
return_fields=return_fields,
enable_scroll=enable_scroll)
except ValueError as e:
abort(HTTP_STATUS_CODE_BAD_REQUEST, e)
# Get number of matching documents per index.
count_per_index = {}
try:
for bucket in result['aggregations']['indices']['buckets']:
key = bucket.get('key')
if key:
count_per_index[key] = bucket.get('doc_count')
except KeyError:
pass
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result['hits']['hits']:
event['selected'] = False
event['_source']['label'] = []
try:
for label in event['_source']['timesketch_label']:
if sketch.id != label['sketch_id']:
continue
event['_source']['label'].append(label['name'])
del event['_source']['timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
# TODO: Add a call to utils.update_sketch_last_activity once new
# mechanism has been added, instead of relying on user views.
view = View.get_or_create(user=current_user, sketch=sketch, name='')
view.update_modification_time()
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
'es_time': result['took'],
'es_total_count': result['hits']['total'],
'timeline_colors': tl_colors,
'timeline_names': tl_names,
'count_per_index': count_per_index,
'scroll_id': result.get('_scroll_id', ''),
}
# Elasticsearch version 7.x returns total hits as a dictionary.
# TODO: Refactor when version 6.x has been deprecated.
if isinstance(meta['es_total_count'], dict):
meta['es_total_count'] = meta['es_total_count'].get('value', 0)
schema = {'meta': meta, 'objects': result['hits']['hits']}
return jsonify(schema)
def get(self, sketch_id):
"""Handles GET request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
args = self.parser.parse_args()
sketch = Sketch.query.get_with_acl(sketch_id)
query_filter = json.loads(args.get(u'filter'))
sketch_indices = [t.searchindex.index_name for t in sketch.timelines]
indices = query_filter.get(u'indices', sketch_indices)
# Make sure that the indices in the filter is part of the sketch
if set(indices) - set(sketch_indices):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Make sure we have a query string or star filter
if not args.get(u'q') and not query_filter.get(u'star'):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
result = self.datastore.search(sketch_id, args[u'q'], query_filter,
indices)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result[u'hits'][u'hits']:
event[u'_source'][u'label'] = []
try:
for label in event[u'_source'][u'timesketch_label']:
if sketch.id != label[u'sketch_id']:
continue
event[u'_source'][u'label'].append(label[u'name'])
del event[u'_source'][u'timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let the
# user get back to the last state in the explore view.
view = View.get_or_create(user=current_user,
sketch=sketch,
name=u'',
query_string=u'',
query_filter=u'')
view.query_string = args.get(u'q')
view.query_filter = args.get(u'filter')
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to render
# the event correctly and to display timing and hit count information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
u'es_time': result[u'took'],
u'es_total_count': result[u'hits'][u'total'],
u'timeline_colors': tl_colors,
u'timeline_names': tl_names
}
schema = {u'meta': meta, u'objects': result[u'hits'][u'hits']}
return jsonify(schema)
