def test_get_emojis_as_dict(self):
"""Test getting emojis as a dictionary."""
skull_code = '☠'
helper_text = 'Suspicious entry'
emojis_dict = emojis.get_emojis_as_dict()
skull_helper = emojis_dict.get(skull_code)
self.assertEqual(skull_helper, helper_text)
does_not_exist = emojis_dict.get('finns_inte')
self.assertIsNone(does_not_exist)
def test_get_emojis_as_dict(self):
"""Test getting emojis as a dictionary."""
skull_code = '☠'
helper_text = 'Suspicious entry'
emojis_dict = emojis.get_emojis_as_dict()
skull_helper = emojis_dict.get(skull_code)
self.assertEqual(skull_helper, helper_text)
does_not_exist = emojis_dict.get('finns_inte')
self.assertIsNone(does_not_exist)
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, 'read'):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
'form_fields': cls.FORM_FIELDS,
'display_name': cls.DISPLAY_NAME,
'description': cls.DESCRIPTION
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name for t in sketch.active_timelines
if t.get_status.status != 'archived'
]
# Get event count and size on disk for each index in the sketch.
stats_per_index = {}
for timeline in sketch.active_timelines:
if timeline.get_status.status != 'archived':
continue
stats_per_index[timeline.searchindex.index_name] = {
'count': 0,
'bytes': 0,
'data_types': []
}
if sketch_indices:
try:
es_stats = self.datastore.client.indices.stats(
index=sketch_indices, metric='docs, store')
except elasticsearch.NotFoundError:
es_stats = {}
logger.error('Unable to find index in datastore',
exc_info=True)
# Stats for index. Num docs per shard and size on disk.
for index_name, stats in es_stats.get('indices', {}).items():
doc_count_all_shards = stats.get('total',
{}).get('docs',
{}).get('count', 0)
bytes_on_disk = stats.get('total',
{}).get('store',
{}).get('size_in_bytes', 0)
num_shards = stats.get('_shards', {}).get('total', 1)
doc_count = int(doc_count_all_shards / num_shards)
stats_per_index[index_name] = {
'count': doc_count,
'bytes': bytes_on_disk
}
# Stats per data type in the index.
parameters = {'limit': '100', 'field': 'data_type'}
result_obj, _ = utils.run_aggregator(
sketch.id,
aggregator_name='field_bucket',
aggregator_parameters=parameters,
index=[index_name])
stats_per_index[index_name]['data_types'] = result_obj.values
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except elasticsearch.NotFoundError:
logger.error('Unable to get indices mapping in datastore',
exc_info=True)
mappings_settings = {}
mappings = []
for _, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value['mappings'].get('properties')
if not properties:
properties = next(iter(
value['mappings'].values())).get('properties')
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith('__'):
continue
if field == 'timesketch_label':
continue
mapping_dict['field'] = field
mapping_dict['type'] = value_dict.get('type', 'n/a')
mappings.append(mapping_dict)
# Make the list of dicts unique
mappings = {v['field']: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = '******'
else:
username = view.user.username
view = {
'name': view.name,
'description': view.description,
'id': view.id,
'query': view.query_string,
'filter': view.query_filter,
'user': username,
'created_at': view.created_at,
'updated_at': view.updated_at
}
views.append(view)
meta = dict(
aggregators=aggregators,
views=views,
searchtemplates=[{
'name': searchtemplate.name,
'id': searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
'public': bool(sketch.is_public),
'read': bool(sketch.has_permission(current_user, 'read')),
'write': bool(sketch.has_permission(current_user, 'write')),
'delete': bool(sketch.has_permission(current_user, 'delete')),
},
collaborators={
'users': [user.username for user in sketch.collaborators],
'groups': [group.name for group in sketch.groups],
},
analyzers=[
x for x, y in analyzer_manager.AnalysisManager.get_analyzers()
],
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
stats=stats_per_index,
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels])
return self.to_json(sketch, meta=meta)
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, 'read'):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
'form_fields': cls.FORM_FIELDS,
'display_name': cls.DISPLAY_NAME,
'description': cls.DESCRIPTION
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name for t in sketch.active_timelines
if t.get_status.status != 'archived'
]
# Get event count and size on disk for each index in the sketch.
indices_metadata = {}
stats_per_timeline = {}
for timeline in sketch.active_timelines:
indices_metadata[timeline.searchindex.index_name] = {}
stats_per_timeline[timeline.id] = {'count': 0}
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except elasticsearch.NotFoundError:
logger.error('Unable to get indices mapping in datastore, for '
'indices: {0:s}'.format(','.join(sketch_indices)))
mappings_settings = {}
mappings = []
for index_name, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value['mappings'].get('properties')
if not properties:
properties = next(iter(
value['mappings'].values())).get('properties')
# Determine if index is from the time before multiple timelines per
# index. This is used in the UI to support both modes.
is_legacy = bool('__ts_timeline_id' not in properties)
indices_metadata[index_name]['is_legacy'] = is_legacy
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith('__'):
continue
if field == 'timesketch_label':
continue
mapping_dict['field'] = field
mapping_dict['type'] = value_dict.get('type', 'n/a')
mappings.append(mapping_dict)
if sketch_indices:
# Stats for index. Num docs per shard.
for timeline in sketch.active_timelines:
index_name = timeline.searchindex.index_name
if indices_metadata[index_name].get('is_legacy', False):
doc_count, _ = self.datastore.count(indices=index_name)
stats_per_timeline[timeline.id] = {'count': doc_count}
else:
# TODO: Consider using an aggregation here instead?
query_dsl = {
'query': {
'term': {
'__ts_timeline_id': {
'value': timeline.id
}
}
}
}
try:
searchindex_name = timeline.searchindex.index_name
except elasticsearch.NotFoundError:
searchindex_name = ''
count = self.datastore.search(sketch_id=sketch.id,
query_string=None,
query_filter={},
query_dsl=query_dsl,
indices=[searchindex_name],
count=True)
stats_per_timeline[timeline.id] = {'count': count}
# Make the list of dicts unique
mappings = {v['field']: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = '******'
else:
username = view.user.username
view = {
'name': view.name,
'description': view.description,
'id': view.id,
'query': view.query_string,
'filter': view.query_filter,
'user': username,
'created_at': view.created_at,
'updated_at': view.updated_at
}
views.append(view)
views.sort(key=lambda x: x.get('name', 'Z').lower())
stories = []
for story in sketch.stories:
if not story.user:
username = '******'
else:
username = story.user.username
story = {
'id': story.id,
'title': story.title,
'user': username,
'created_at': story.created_at,
'updated_at': story.updated_at
}
stories.append(story)
meta = dict(aggregators=aggregators,
views=views,
stories=stories,
searchtemplates=[{
'name': searchtemplate.name,
'id': searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
'public':
bool(sketch.is_public),
'read':
bool(sketch.has_permission(current_user, 'read')),
'write':
bool(sketch.has_permission(current_user, 'write')),
'delete':
bool(sketch.has_permission(current_user, 'delete')),
},
collaborators={
'users':
[user.username for user in sketch.collaborators],
'groups': [group.name for group in sketch.groups],
},
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
indices_metadata=indices_metadata,
stats_per_timeline=stats_per_timeline,
last_activity=utils.get_sketch_last_activity(sketch),
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels])
return self.to_json(sketch, meta=meta)
def get(self, sketch_id):
"""Handles GET request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
if current_user.admin:
sketch = Sketch.query.get(sketch_id)
if not sketch.has_permission(current_user, "read"):
return self._get_sketch_for_admin(sketch)
else:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
aggregators = {}
for _, cls in aggregator_manager.AggregatorManager.get_aggregators():
aggregators[cls.NAME] = {
"form_fields": cls.FORM_FIELDS,
"display_name": cls.DISPLAY_NAME,
"description": cls.DESCRIPTION,
}
# Get mappings for all indices in the sketch. This is used to set
# columns shown in the event list.
sketch_indices = [
t.searchindex.index_name for t in sketch.active_timelines
]
# Make sure the list of index names is uniq
sketch_indices = list(set(sketch_indices))
# Get event count and size on disk for each index in the sketch.
indices_metadata = {}
stats_per_timeline = {}
for timeline in sketch.active_timelines:
indices_metadata[timeline.searchindex.index_name] = {}
stats_per_timeline[timeline.id] = {"count": 0}
if not sketch_indices:
mappings_settings = {}
else:
try:
mappings_settings = self.datastore.client.indices.get_mapping(
index=sketch_indices)
except opensearchpy.NotFoundError:
logger.error("Unable to get indices mapping in datastore, for "
"indices: {0:s}".format(",".join(sketch_indices)))
mappings_settings = {}
mappings = []
for index_name, value in mappings_settings.items():
# The structure is different in ES version 6.x and lower. This check
# makes sure we support both old and new versions.
properties = value["mappings"].get("properties")
if not properties:
properties = next(iter(
value["mappings"].values())).get("properties")
# Determine if index is from the time before multiple timelines per
# index. This is used in the UI to support both modes.
is_legacy = bool("__ts_timeline_id" not in properties)
indices_metadata[index_name]["is_legacy"] = is_legacy
for field, value_dict in properties.items():
mapping_dict = {}
# Exclude internal fields
if field.startswith("__"):
continue
if field == "timesketch_label":
continue
mapping_dict["field"] = field
mapping_dict["type"] = value_dict.get("type", "n/a")
mappings.append(mapping_dict)
# Get number of events per timeline
if sketch_indices:
# Support legacy indices.
for timeline in sketch.active_timelines:
index_name = timeline.searchindex.index_name
if indices_metadata[index_name].get("is_legacy", False):
doc_count, _ = self.datastore.count(indices=index_name)
stats_per_timeline[timeline.id] = {"count": doc_count}
count_agg_spec = {
"aggs": {
"per_timeline": {
"terms": {
"field": "__ts_timeline_id",
"size": len(sketch.timelines),
}
}
}
}
# pylint: disable=unexpected-keyword-arg, no-value-for-parameter
count_agg = self.datastore.client.search(index=sketch_indices,
body=count_agg_spec,
size=0)
count_per_timeline = (count_agg.get("aggregations",
{}).get("per_timeline",
{}).get("buckets", []))
for count_stat in count_per_timeline:
stats_per_timeline[count_stat["key"]] = {
"count": count_stat["doc_count"]
}
# Make the list of dicts unique
mappings = {v["field"]: v for v in mappings}.values()
views = []
for view in sketch.get_named_views:
if not view.user:
username = "******"
else:
username = view.user.username
view = {
"name": view.name,
"description": view.description,
"id": view.id,
"query": view.query_string,
"filter": view.query_filter,
"user": username,
"created_at": view.created_at,
"updated_at": view.updated_at,
}
views.append(view)
views.sort(key=lambda x: x.get("name", "Z").lower())
stories = []
for story in sketch.stories:
if not story.user:
username = "******"
else:
username = story.user.username
story = {
"id": story.id,
"title": story.title,
"user": username,
"created_at": story.created_at,
"updated_at": story.updated_at,
}
stories.append(story)
meta = dict(
aggregators=aggregators,
views=views,
stories=stories,
searchtemplates=[{
"name": searchtemplate.name,
"id": searchtemplate.id
} for searchtemplate in SearchTemplate.query.all()],
emojis=get_emojis_as_dict(),
permissions={
"public": bool(sketch.is_public),
"read": bool(sketch.has_permission(current_user, "read")),
"write": bool(sketch.has_permission(current_user, "write")),
"delete": bool(sketch.has_permission(current_user, "delete")),
},
collaborators={
"users": [user.username for user in sketch.collaborators],
"groups": [group.name for group in sketch.groups],
},
attributes=utils.get_sketch_attributes(sketch),
mappings=list(mappings),
indices_metadata=indices_metadata,
stats_per_timeline=stats_per_timeline,
last_activity=utils.get_sketch_last_activity(sketch),
filter_labels=self.datastore.get_filter_labels(
sketch.id, sketch_indices),
sketch_labels=[label.label for label in sketch.labels],
)
return self.to_json(sketch, meta=meta)
