def _get_sketch_for_admin(self, sketch):
"""Returns a limited sketch view for adminstrators.
An administrator needs to get information about all sketches
that are stored on the backend. However that view should be
limited for sketches that user does not have explicit read
or other permissions as well. In those cases the returned
sketch only contains information about the name, description,
etc but not any information about the data, nor any access
to the underlying data of the sketch.
Args:
sketch: a sketch object (instance of models.Sketch)
Returns:
A limited view of a sketch in JSON (instance of
flask.wrappers.Response)
"""
if sketch.get_status.status == 'archived':
status = 'archived'
else:
status = 'admin_view'
sketch_fields = {
'id': sketch.id,
'name': sketch.name,
'description': sketch.description,
'user': {'username': current_user.username},
'timelines': [],
'stories': [],
'aggregations': [],
'aggregationgroups': [],
'active_timelines': [],
'label_string': sketch.label_string,
'status': [{
'id': 0,
'status': status}],
'all_permissions': sketch.all_permissions,
'created_at': sketch.created_at,
'updated_at': sketch.updated_at,
}
meta = {
'current_user': current_user.username,
'last_activity': utils.get_sketch_last_activity(sketch),
}
return jsonify(
{
'objects': [sketch_fields],
'meta': meta
}
)
def _get_sketch_for_admin(sketch):
"""Returns a limited sketch view for administrators.
An administrator needs to get information about all sketches
that are stored on the backend. However that view should be
limited for sketches that user does not have explicit read
or other permissions as well. In those cases the returned
sketch only contains information about the name, description,
etc but not any information about the data, nor any access
to the underlying data of the sketch.
Args:
sketch: a sketch object (instance of models.Sketch)
Returns:
A limited view of a sketch in JSON (instance of
flask.wrappers.Response)
"""
if sketch.get_status.status == "archived":
status = "archived"
else:
status = "admin_view"
sketch_fields = {
"id": sketch.id,
"name": sketch.name,
"description": sketch.description,
"user": {
"username": current_user.username
},
"timelines": [],
"stories": [],
"active_timelines": [],
"label_string": sketch.label_string,
"status": [{
"id": 0,
"status": status
}],
"all_permissions": sketch.all_permissions,
"created_at": sketch.created_at,
"updated_at": sketch.updated_at,
}
meta = {
"current_user": current_user.username,
"last_activity": utils.get_sketch_last_activity(sketch),
}
return jsonify({"objects": [sketch_fields], "meta": meta})
def get(self):
"""Handles GET request to the resource.
Returns:
List of sketches (instance of flask.wrappers.Response)
"""
args = self.parser.parse_args()
scope = args.get('scope')
page = args.get('page')
per_page = args.get('per_page')
search_query = args.get('search_query')
include_archived = args.get('include_archived')
if current_user.admin and scope == 'admin':
sketch_query = Sketch.query
else:
sketch_query = Sketch.all_with_acl()
base_filter = sketch_query.filter(
not_(Sketch.Status.status == 'deleted'),
not_(Sketch.Status.status == 'archived'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
base_filter_with_archived = sketch_query.filter(
not_(Sketch.Status.status == 'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
filtered_sketches = base_filter_with_archived
sketches = []
return_sketches = []
has_next = False
has_prev = False
next_page = None
prev_page = None
current_page = 1
total_pages = 0
total_items = 0
if scope == 'recent':
# Get list of sketches that the user has actively searched in.
# TODO: Make this cover more actions such as story updates etc.
# TODO: Right now we only return the top 3, make this configurable.
views = View.query.filter_by(
user=current_user, name='').order_by(
View.updated_at.desc()).limit(3)
sketches = [view.sketch for view in views]
total_items = len(sketches)
elif scope == 'admin':
if not current_user.admin:
abort(HTTP_STATUS_CODE_FORBIDDEN, 'User is not an admin.')
if include_archived:
filtered_sketches = base_filter_with_archived
else:
filtered_sketches = base_filter
elif scope == 'user':
filtered_sketches = base_filter.filter_by(user=current_user)
elif scope == 'archived':
filtered_sketches = sketch_query.filter(
Sketch.status.any(status='archived'))
elif scope == 'shared':
filtered_sketches = base_filter.filter(Sketch.user != current_user)
elif scope == 'search':
filtered_sketches = base_filter_with_archived.filter(
or_(
Sketch.name.ilike(f'%{search_query}%'),
Sketch.description.ilike(f'%{search_query}%')
)
)
if not sketches:
pagination = filtered_sketches.paginate(
page=page, per_page=per_page)
sketches = pagination.items
has_next = pagination.has_next
has_prev = pagination.has_prev
next_page = pagination.next_num
prev_page = pagination.prev_num
current_page = pagination.page
total_pages = pagination.pages
total_items = pagination.total
for sketch in sketches:
# Return a subset of the sketch objects to reduce the amount of
# data sent to the client.
return_sketches.append({
'id': sketch.id,
'name': sketch.name,
'description': sketch.description,
'created_at': str(sketch.created_at),
'last_activity': utils.get_sketch_last_activity(sketch),
'user': sketch.user.username,
'status': sketch.get_status.status
})
meta = {
'current_user': current_user.username,
'has_next': has_next,
'has_prev': has_prev,
'next_page': next_page,
'prev_page': prev_page,
'current_page': current_page,
'total_pages': total_pages,
'total_items': total_items
}
return jsonify({'objects': return_sketches, 'meta': meta})
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, 'read'):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
'form_fields': cls.FORM_FIELDS,
'display_name': cls.DISPLAY_NAME,
'description': cls.DESCRIPTION
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name
for t in sketch.active_timelines
if t.get_status.status != 'archived'
]
# Get event count and size on disk for each index in the sketch.
stats_per_index = {}
for timeline in sketch.active_timelines:
if timeline.get_status.status != 'archived':
continue
stats_per_index[timeline.searchindex.index_name] = {
'count': 0,
'bytes': 0,
'data_types': []
}
if sketch_indices:
# Stats for index. Num docs per shard and size on disk.
for index_name in sketch_indices:
doc_count, bytes_on_disk = self.datastore.count(
indices=index_name)
stats_per_index[index_name] = {
'count': doc_count,
'bytes': bytes_on_disk
}
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except elasticsearch.NotFoundError:
logger.error(
'Unable to get indices mapping in datastore', exc_info=True)
mappings_settings = {}
mappings = []
for _, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value['mappings'].get('properties')
if not properties:
properties = next(
iter(value['mappings'].values())).get('properties')
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith('__'):
continue
if field == 'timesketch_label':
continue
mapping_dict['field'] = field
mapping_dict['type'] = value_dict.get('type', 'n/a')
mappings.append(mapping_dict)
# Make the list of dicts unique
mappings = {v['field']: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = '******'
else:
username = view.user.username
view = {
'name': view.name,
'description': view.description,
'id': view.id,
'query': view.query_string,
'filter': view.query_filter,
'user': username,
'created_at': view.created_at,
'updated_at': view.updated_at
}
views.append(view)
stories = []
for story in sketch.stories:
if not story.user:
username = '******'
else:
username = story.user.username
story = {
'id': story.id,
'title': story.title,
'user': username,
'created_at': story.created_at,
'updated_at': story.updated_at
}
stories.append(story)
meta = dict(
aggregators=aggregators,
views=views,
stories=stories,
searchtemplates=[{
'name': searchtemplate.name,
'id': searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
'public': bool(sketch.is_public),
'read': bool(sketch.has_permission(current_user, 'read')),
'write': bool(sketch.has_permission(current_user, 'write')),
'delete': bool(sketch.has_permission(current_user, 'delete')),
},
collaborators={
'users': [user.username for user in sketch.collaborators],
'groups': [group.name for group in sketch.groups],
},
analyzers=[
x for x, y in analyzer_manager.AnalysisManager.get_analyzers()
],
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
stats=stats_per_index,
last_activity=utils.get_sketch_last_activity(sketch),
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels]
)
return self.to_json(sketch, meta=meta)
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, 'read'):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
'form_fields': cls.FORM_FIELDS,
'display_name': cls.DISPLAY_NAME,
'description': cls.DESCRIPTION
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name for t in sketch.active_timelines
]
# Make sure the list of index names is uniq
sketch_indices = list(set(sketch_indices))
# Get event count and size on disk for each index in the sketch.
indices_metadata = {}
stats_per_timeline = {}
for timeline in sketch.active_timelines:
indices_metadata[timeline.searchindex.index_name] = {}
stats_per_timeline[timeline.id] = {'count': 0}
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except elasticsearch.NotFoundError:
logger.error('Unable to get indices mapping in datastore, for '
'indices: {0:s}'.format(','.join(sketch_indices)))
mappings_settings = {}
mappings = []
for index_name, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value['mappings'].get('properties')
if not properties:
properties = next(iter(
value['mappings'].values())).get('properties')
# Determine if index is from the time before multiple timelines per
# index. This is used in the UI to support both modes.
is_legacy = bool('__ts_timeline_id' not in properties)
indices_metadata[index_name]['is_legacy'] = is_legacy
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith('__'):
continue
if field == 'timesketch_label':
continue
mapping_dict['field'] = field
mapping_dict['type'] = value_dict.get('type', 'n/a')
mappings.append(mapping_dict)
# Get number of events per timeline
if sketch_indices:
# Support legacy indices.
for timeline in sketch.active_timelines:
index_name = timeline.searchindex.index_name
if indices_metadata[index_name].get('is_legacy', False):
doc_count, _ = self.datastore.count(indices=index_name)
stats_per_timeline[timeline.id] = {'count': doc_count}
count_agg_spec = {
'aggs': {
'per_timeline': {
'terms': {
'field': '__ts_timeline_id',
'size': len(sketch.timelines),
}
}
}
}
# pylint: disable=unexpected-keyword-arg, no-value-for-parameter
count_agg = self.datastore.client.search(index=sketch_indices,
body=count_agg_spec,
size=0)
count_per_timeline = count_agg.get('aggregations',
{}).get('per_timeline',
{}).get('buckets', [])
for count_stat in count_per_timeline:
stats_per_timeline[count_stat['key']] = {
'count': count_stat['doc_count']
}
# Make the list of dicts unique
mappings = {v['field']: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = '******'
else:
username = view.user.username
view = {
'name': view.name,
'description': view.description,
'id': view.id,
'query': view.query_string,
'filter': view.query_filter,
'user': username,
'created_at': view.created_at,
'updated_at': view.updated_at
}
views.append(view)
views.sort(key=lambda x: x.get('name', 'Z').lower())
stories = []
for story in sketch.stories:
if not story.user:
username = '******'
else:
username = story.user.username
story = {
'id': story.id,
'title': story.title,
'user': username,
'created_at': story.created_at,
'updated_at': story.updated_at
}
stories.append(story)
meta = dict(aggregators=aggregators,
views=views,
stories=stories,
searchtemplates=[{
'name': searchtemplate.name,
'id': searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
'public':
bool(sketch.is_public),
'read':
bool(sketch.has_permission(current_user, 'read')),
'write':
bool(sketch.has_permission(current_user, 'write')),
'delete':
bool(sketch.has_permission(current_user, 'delete')),
},
collaborators={
'users':
[user.username for user in sketch.collaborators],
'groups': [group.name for group in sketch.groups],
},
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
indices_metadata=indices_metadata,
stats_per_timeline=stats_per_timeline,
last_activity=utils.get_sketch_last_activity(sketch),
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels])
return self.to_json(sketch, meta=meta)
def get(self):
"""Handles GET request to the resource.
Returns:
List of sketches (instance of flask.wrappers.Response)
"""
args = self.parser.parse_args()
scope = args.get("scope")
page = args.get("page")
per_page = args.get("per_page")
search_query = args.get("search_query")
include_archived = args.get("include_archived")
if current_user.admin and scope == "admin":
sketch_query = Sketch.query
else:
sketch_query = Sketch.all_with_acl()
base_filter = sketch_query.filter(
not_(Sketch.Status.status == "deleted"),
not_(Sketch.Status.status == "archived"),
Sketch.Status.parent,
).order_by(Sketch.updated_at.desc())
base_filter_with_archived = sketch_query.filter(
not_(Sketch.Status.status == "deleted"),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
filtered_sketches = base_filter_with_archived
sketches = []
return_sketches = []
has_next = False
has_prev = False
next_page = None
prev_page = None
current_page = 1
total_pages = 0
total_items = 0
if scope == "recent":
# Get list of sketches that the user has actively searched in.
# TODO: Make this cover more actions such as story updates etc.
# TODO: Right now we only return the top 3, make this configurable.
views = (View.query.filter_by(user=current_user, name="").order_by(
View.updated_at.desc()).limit(3))
sketches = [view.sketch for view in views]
total_items = len(sketches)
elif scope == "admin":
if not current_user.admin:
abort(HTTP_STATUS_CODE_FORBIDDEN, "User is not an admin.")
if include_archived:
filtered_sketches = base_filter_with_archived
else:
filtered_sketches = base_filter
elif scope == "user":
filtered_sketches = base_filter.filter_by(user=current_user)
elif scope == "archived":
filtered_sketches = sketch_query.filter(
Sketch.status.any(status="archived"))
elif scope == "shared":
filtered_sketches = base_filter.filter(Sketch.user != current_user)
elif scope == "search":
filtered_sketches = base_filter_with_archived.filter(
or_(
Sketch.name.ilike(f"%{search_query}%"),
Sketch.description.ilike(f"%{search_query}%"),
))
if not sketches:
pagination = filtered_sketches.paginate(page=page,
per_page=per_page)
sketches = pagination.items
has_next = pagination.has_next
has_prev = pagination.has_prev
next_page = pagination.next_num
prev_page = pagination.prev_num
current_page = pagination.page
total_pages = pagination.pages
total_items = pagination.total
for sketch in sketches:
# Return a subset of the sketch objects to reduce the amount of
# data sent to the client.
return_sketches.append({
"id":
sketch.id,
"name":
sketch.name,
"description":
sketch.description,
"created_at":
str(sketch.created_at),
"last_activity":
utils.get_sketch_last_activity(sketch),
"user":
sketch.user.username,
"status":
sketch.get_status.status,
})
meta = {
"current_user": current_user.username,
"has_next": has_next,
"has_prev": has_prev,
"next_page": next_page,
"prev_page": prev_page,
"current_page": current_page,
"total_pages": total_pages,
"total_items": total_items,
}
return jsonify({"objects": return_sketches, "meta": meta})
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, "read"):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
"form_fields": cls.FORM_FIELDS,
"display_name": cls.DISPLAY_NAME,
"description": cls.DESCRIPTION,
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name for t in sketch.active_timelines
]
# Make sure the list of index names is uniq
sketch_indices = list(set(sketch_indices))
# Get event count and size on disk for each index in the sketch.
indices_metadata = {}
stats_per_timeline = {}
for timeline in sketch.active_timelines:
indices_metadata[timeline.searchindex.index_name] = {}
stats_per_timeline[timeline.id] = {"count": 0}
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except opensearchpy.NotFoundError:
logger.error("Unable to get indices mapping in datastore, for "
"indices: {0:s}".format(",".join(sketch_indices)))
mappings_settings = {}
mappings = []
for index_name, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value["mappings"].get("properties")
if not properties:
properties = next(iter(
value["mappings"].values())).get("properties")
# Determine if index is from the time before multiple timelines per
# index. This is used in the UI to support both modes.
is_legacy = bool("__ts_timeline_id" not in properties)
indices_metadata[index_name]["is_legacy"] = is_legacy
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith("__"):
continue
if field == "timesketch_label":
continue
mapping_dict["field"] = field
mapping_dict["type"] = value_dict.get("type", "n/a")
mappings.append(mapping_dict)
# Get number of events per timeline
if sketch_indices:
# Support legacy indices.
for timeline in sketch.active_timelines:
index_name = timeline.searchindex.index_name
if indices_metadata[index_name].get("is_legacy", False):
doc_count, _ = self.datastore.count(indices=index_name)
stats_per_timeline[timeline.id] = {"count": doc_count}
count_agg_spec = {
"aggs": {
"per_timeline": {
"terms": {
"field": "__ts_timeline_id",
"size": len(sketch.timelines),
}
}
}
}
# pylint: disable=unexpected-keyword-arg, no-value-for-parameter
count_agg = self.datastore.client.search(index=sketch_indices,
body=count_agg_spec,
size=0)
count_per_timeline = (count_agg.get("aggregations",
{}).get("per_timeline",
{}).get("buckets", []))
for count_stat in count_per_timeline:
stats_per_timeline[count_stat["key"]] = {
"count": count_stat["doc_count"]
}
# Make the list of dicts unique
mappings = {v["field"]: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = "******"
else:
username = view.user.username
view = {
"name": view.name,
"description": view.description,
"id": view.id,
"query": view.query_string,
"filter": view.query_filter,
"user": username,
"created_at": view.created_at,
"updated_at": view.updated_at,
}
views.append(view)
views.sort(key=lambda x: x.get("name", "Z").lower())
stories = []
for story in sketch.stories:
if not story.user:
username = "******"
else:
username = story.user.username
story = {
"id": story.id,
"title": story.title,
"user": username,
"created_at": story.created_at,
"updated_at": story.updated_at,
}
stories.append(story)
meta = dict(
aggregators=aggregators,
views=views,
stories=stories,
searchtemplates=[{
"name": searchtemplate.name,
"id": searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
"public": bool(sketch.is_public),
"read": bool(sketch.has_permission(current_user, "read")),
"write": bool(sketch.has_permission(current_user, "write")),
"delete": bool(sketch.has_permission(current_user, "delete")),
},
collaborators={
"users": [user.username for user in sketch.collaborators],
"groups": [group.name for group in sketch.groups],
},
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
indices_metadata=indices_metadata,
stats_per_timeline=stats_per_timeline,
last_activity=utils.get_sketch_last_activity(sketch),
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels],
)
return self.to_json(sketch, meta=meta)
