def _get_sketch_for_admin(self, sketch): """Returns a limited sketch view for adminstrators. An administrator needs to get information about all sketches that are stored on the backend. However that view should be limited for sketches that user does not have explicit read or other permissions as well. In those cases the returned sketch only contains information about the name, description, etc but not any information about the data, nor any access to the underlying data of the sketch. Args: sketch: a sketch object (instance of models.Sketch) Returns: A limited view of a sketch in JSON (instance of flask.wrappers.Response) """ if sketch.get_status.status == 'archived': status = 'archived' else: status = 'admin_view' sketch_fields = { 'id': sketch.id, 'name': sketch.name, 'description': sketch.description, 'user': {'username': current_user.username}, 'timelines': [], 'stories': [], 'aggregations': [], 'aggregationgroups': [], 'active_timelines': [], 'label_string': sketch.label_string, 'status': [{ 'id': 0, 'status': status}], 'all_permissions': sketch.all_permissions, 'created_at': sketch.created_at, 'updated_at': sketch.updated_at, } meta = { 'current_user': current_user.username, 'last_activity': utils.get_sketch_last_activity(sketch), } return jsonify( { 'objects': [sketch_fields], 'meta': meta } )
def _get_sketch_for_admin(sketch): """Returns a limited sketch view for administrators. An administrator needs to get information about all sketches that are stored on the backend. However that view should be limited for sketches that user does not have explicit read or other permissions as well. In those cases the returned sketch only contains information about the name, description, etc but not any information about the data, nor any access to the underlying data of the sketch. Args: sketch: a sketch object (instance of models.Sketch) Returns: A limited view of a sketch in JSON (instance of flask.wrappers.Response) """ if sketch.get_status.status == "archived": status = "archived" else: status = "admin_view" sketch_fields = { "id": sketch.id, "name": sketch.name, "description": sketch.description, "user": { "username": current_user.username }, "timelines": [], "stories": [], "active_timelines": [], "label_string": sketch.label_string, "status": [{ "id": 0, "status": status }], "all_permissions": sketch.all_permissions, "created_at": sketch.created_at, "updated_at": sketch.updated_at, } meta = { "current_user": current_user.username, "last_activity": utils.get_sketch_last_activity(sketch), } return jsonify({"objects": [sketch_fields], "meta": meta})
def get(self): """Handles GET request to the resource. Returns: List of sketches (instance of flask.wrappers.Response) """ args = self.parser.parse_args() scope = args.get('scope') page = args.get('page') per_page = args.get('per_page') search_query = args.get('search_query') include_archived = args.get('include_archived') if current_user.admin and scope == 'admin': sketch_query = Sketch.query else: sketch_query = Sketch.all_with_acl() base_filter = sketch_query.filter( not_(Sketch.Status.status == 'deleted'), not_(Sketch.Status.status == 'archived'), Sketch.Status.parent).order_by(Sketch.updated_at.desc()) base_filter_with_archived = sketch_query.filter( not_(Sketch.Status.status == 'deleted'), Sketch.Status.parent).order_by(Sketch.updated_at.desc()) filtered_sketches = base_filter_with_archived sketches = [] return_sketches = [] has_next = False has_prev = False next_page = None prev_page = None current_page = 1 total_pages = 0 total_items = 0 if scope == 'recent': # Get list of sketches that the user has actively searched in. # TODO: Make this cover more actions such as story updates etc. # TODO: Right now we only return the top 3, make this configurable. views = View.query.filter_by( user=current_user, name='').order_by( View.updated_at.desc()).limit(3) sketches = [view.sketch for view in views] total_items = len(sketches) elif scope == 'admin': if not current_user.admin: abort(HTTP_STATUS_CODE_FORBIDDEN, 'User is not an admin.') if include_archived: filtered_sketches = base_filter_with_archived else: filtered_sketches = base_filter elif scope == 'user': filtered_sketches = base_filter.filter_by(user=current_user) elif scope == 'archived': filtered_sketches = sketch_query.filter( Sketch.status.any(status='archived')) elif scope == 'shared': filtered_sketches = base_filter.filter(Sketch.user != current_user) elif scope == 'search': filtered_sketches = base_filter_with_archived.filter( or_( Sketch.name.ilike(f'%{search_query}%'), Sketch.description.ilike(f'%{search_query}%') ) ) if not sketches: pagination = filtered_sketches.paginate( page=page, per_page=per_page) sketches = pagination.items has_next = pagination.has_next has_prev = pagination.has_prev next_page = pagination.next_num prev_page = pagination.prev_num current_page = pagination.page total_pages = pagination.pages total_items = pagination.total for sketch in sketches: # Return a subset of the sketch objects to reduce the amount of # data sent to the client. return_sketches.append({ 'id': sketch.id, 'name': sketch.name, 'description': sketch.description, 'created_at': str(sketch.created_at), 'last_activity': utils.get_sketch_last_activity(sketch), 'user': sketch.user.username, 'status': sketch.get_status.status }) meta = { 'current_user': current_user.username, 'has_next': has_next, 'has_prev': has_prev, 'next_page': next_page, 'prev_page': prev_page, 'current_page': current_page, 'total_pages': total_pages, 'total_items': total_items } return jsonify({'objects': return_sketches, 'meta': meta})
def get(self, sketch_id): """Handles GET request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ if current_user.admin: sketch = Sketch.query.get(sketch_id) if not sketch.has_permission(current_user, 'read'): return self._get_sketch_for_admin(sketch) else: sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') aggregators = {} for _, cls in aggregator_manager.AggregatorManager.get_aggregators(): aggregators[cls.NAME] = { 'form_fields': cls.FORM_FIELDS, 'display_name': cls.DISPLAY_NAME, 'description': cls.DESCRIPTION } # Get mappings for all indices in the sketch. This is used to set # columns shown in the event list. sketch_indices = [ t.searchindex.index_name for t in sketch.active_timelines if t.get_status.status != 'archived' ] # Get event count and size on disk for each index in the sketch. stats_per_index = {} for timeline in sketch.active_timelines: if timeline.get_status.status != 'archived': continue stats_per_index[timeline.searchindex.index_name] = { 'count': 0, 'bytes': 0, 'data_types': [] } if sketch_indices: # Stats for index. Num docs per shard and size on disk. for index_name in sketch_indices: doc_count, bytes_on_disk = self.datastore.count( indices=index_name) stats_per_index[index_name] = { 'count': doc_count, 'bytes': bytes_on_disk } if not sketch_indices: mappings_settings = {} else: try: mappings_settings = self.datastore.client.indices.get_mapping( index=sketch_indices) except elasticsearch.NotFoundError: logger.error( 'Unable to get indices mapping in datastore', exc_info=True) mappings_settings = {} mappings = [] for _, value in mappings_settings.items(): # The structure is different in ES version 6.x and lower. This check # makes sure we support both old and new versions. properties = value['mappings'].get('properties') if not properties: properties = next( iter(value['mappings'].values())).get('properties') for field, value_dict in properties.items(): mapping_dict = {} # Exclude internal fields if field.startswith('__'): continue if field == 'timesketch_label': continue mapping_dict['field'] = field mapping_dict['type'] = value_dict.get('type', 'n/a') mappings.append(mapping_dict) # Make the list of dicts unique mappings = {v['field']: v for v in mappings}.values() views = [] for view in sketch.get_named_views: if not view.user: username = '******' else: username = view.user.username view = { 'name': view.name, 'description': view.description, 'id': view.id, 'query': view.query_string, 'filter': view.query_filter, 'user': username, 'created_at': view.created_at, 'updated_at': view.updated_at } views.append(view) stories = [] for story in sketch.stories: if not story.user: username = '******' else: username = story.user.username story = { 'id': story.id, 'title': story.title, 'user': username, 'created_at': story.created_at, 'updated_at': story.updated_at } stories.append(story) meta = dict( aggregators=aggregators, views=views, stories=stories, searchtemplates=[{ 'name': searchtemplate.name, 'id': searchtemplate.id } for searchtemplate in SearchTemplate.query.all()], emojis=get_emojis_as_dict(), permissions={ 'public': bool(sketch.is_public), 'read': bool(sketch.has_permission(current_user, 'read')), 'write': bool(sketch.has_permission(current_user, 'write')), 'delete': bool(sketch.has_permission(current_user, 'delete')), }, collaborators={ 'users': [user.username for user in sketch.collaborators], 'groups': [group.name for group in sketch.groups], }, analyzers=[ x for x, y in analyzer_manager.AnalysisManager.get_analyzers() ], attributes=utils.get_sketch_attributes(sketch), mappings=list(mappings), stats=stats_per_index, last_activity=utils.get_sketch_last_activity(sketch), filter_labels=self.datastore.get_filter_labels( sketch.id, sketch_indices), sketch_labels=[label.label for label in sketch.labels] ) return self.to_json(sketch, meta=meta)
def get(self, sketch_id): """Handles GET request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ if current_user.admin: sketch = Sketch.query.get(sketch_id) if not sketch.has_permission(current_user, 'read'): return self._get_sketch_for_admin(sketch) else: sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') aggregators = {} for _, cls in aggregator_manager.AggregatorManager.get_aggregators(): aggregators[cls.NAME] = { 'form_fields': cls.FORM_FIELDS, 'display_name': cls.DISPLAY_NAME, 'description': cls.DESCRIPTION } # Get mappings for all indices in the sketch. This is used to set # columns shown in the event list. sketch_indices = [ t.searchindex.index_name for t in sketch.active_timelines ] # Make sure the list of index names is uniq sketch_indices = list(set(sketch_indices)) # Get event count and size on disk for each index in the sketch. indices_metadata = {} stats_per_timeline = {} for timeline in sketch.active_timelines: indices_metadata[timeline.searchindex.index_name] = {} stats_per_timeline[timeline.id] = {'count': 0} if not sketch_indices: mappings_settings = {} else: try: mappings_settings = self.datastore.client.indices.get_mapping( index=sketch_indices) except elasticsearch.NotFoundError: logger.error('Unable to get indices mapping in datastore, for ' 'indices: {0:s}'.format(','.join(sketch_indices))) mappings_settings = {} mappings = [] for index_name, value in mappings_settings.items(): # The structure is different in ES version 6.x and lower. This check # makes sure we support both old and new versions. properties = value['mappings'].get('properties') if not properties: properties = next(iter( value['mappings'].values())).get('properties') # Determine if index is from the time before multiple timelines per # index. This is used in the UI to support both modes. is_legacy = bool('__ts_timeline_id' not in properties) indices_metadata[index_name]['is_legacy'] = is_legacy for field, value_dict in properties.items(): mapping_dict = {} # Exclude internal fields if field.startswith('__'): continue if field == 'timesketch_label': continue mapping_dict['field'] = field mapping_dict['type'] = value_dict.get('type', 'n/a') mappings.append(mapping_dict) # Get number of events per timeline if sketch_indices: # Support legacy indices. for timeline in sketch.active_timelines: index_name = timeline.searchindex.index_name if indices_metadata[index_name].get('is_legacy', False): doc_count, _ = self.datastore.count(indices=index_name) stats_per_timeline[timeline.id] = {'count': doc_count} count_agg_spec = { 'aggs': { 'per_timeline': { 'terms': { 'field': '__ts_timeline_id', 'size': len(sketch.timelines), } } } } # pylint: disable=unexpected-keyword-arg, no-value-for-parameter count_agg = self.datastore.client.search(index=sketch_indices, body=count_agg_spec, size=0) count_per_timeline = count_agg.get('aggregations', {}).get('per_timeline', {}).get('buckets', []) for count_stat in count_per_timeline: stats_per_timeline[count_stat['key']] = { 'count': count_stat['doc_count'] } # Make the list of dicts unique mappings = {v['field']: v for v in mappings}.values() views = [] for view in sketch.get_named_views: if not view.user: username = '******' else: username = view.user.username view = { 'name': view.name, 'description': view.description, 'id': view.id, 'query': view.query_string, 'filter': view.query_filter, 'user': username, 'created_at': view.created_at, 'updated_at': view.updated_at } views.append(view) views.sort(key=lambda x: x.get('name', 'Z').lower()) stories = [] for story in sketch.stories: if not story.user: username = '******' else: username = story.user.username story = { 'id': story.id, 'title': story.title, 'user': username, 'created_at': story.created_at, 'updated_at': story.updated_at } stories.append(story) meta = dict(aggregators=aggregators, views=views, stories=stories, searchtemplates=[{ 'name': searchtemplate.name, 'id': searchtemplate.id } for searchtemplate in SearchTemplate.query.all()], emojis=get_emojis_as_dict(), permissions={ 'public': bool(sketch.is_public), 'read': bool(sketch.has_permission(current_user, 'read')), 'write': bool(sketch.has_permission(current_user, 'write')), 'delete': bool(sketch.has_permission(current_user, 'delete')), }, collaborators={ 'users': [user.username for user in sketch.collaborators], 'groups': [group.name for group in sketch.groups], }, attributes=utils.get_sketch_attributes(sketch), mappings=list(mappings), indices_metadata=indices_metadata, stats_per_timeline=stats_per_timeline, last_activity=utils.get_sketch_last_activity(sketch), filter_labels=self.datastore.get_filter_labels( sketch.id, sketch_indices), sketch_labels=[label.label for label in sketch.labels]) return self.to_json(sketch, meta=meta)
def get(self): """Handles GET request to the resource. Returns: List of sketches (instance of flask.wrappers.Response) """ args = self.parser.parse_args() scope = args.get("scope") page = args.get("page") per_page = args.get("per_page") search_query = args.get("search_query") include_archived = args.get("include_archived") if current_user.admin and scope == "admin": sketch_query = Sketch.query else: sketch_query = Sketch.all_with_acl() base_filter = sketch_query.filter( not_(Sketch.Status.status == "deleted"), not_(Sketch.Status.status == "archived"), Sketch.Status.parent, ).order_by(Sketch.updated_at.desc()) base_filter_with_archived = sketch_query.filter( not_(Sketch.Status.status == "deleted"), Sketch.Status.parent).order_by(Sketch.updated_at.desc()) filtered_sketches = base_filter_with_archived sketches = [] return_sketches = [] has_next = False has_prev = False next_page = None prev_page = None current_page = 1 total_pages = 0 total_items = 0 if scope == "recent": # Get list of sketches that the user has actively searched in. # TODO: Make this cover more actions such as story updates etc. # TODO: Right now we only return the top 3, make this configurable. views = (View.query.filter_by(user=current_user, name="").order_by( View.updated_at.desc()).limit(3)) sketches = [view.sketch for view in views] total_items = len(sketches) elif scope == "admin": if not current_user.admin: abort(HTTP_STATUS_CODE_FORBIDDEN, "User is not an admin.") if include_archived: filtered_sketches = base_filter_with_archived else: filtered_sketches = base_filter elif scope == "user": filtered_sketches = base_filter.filter_by(user=current_user) elif scope == "archived": filtered_sketches = sketch_query.filter( Sketch.status.any(status="archived")) elif scope == "shared": filtered_sketches = base_filter.filter(Sketch.user != current_user) elif scope == "search": filtered_sketches = base_filter_with_archived.filter( or_( Sketch.name.ilike(f"%{search_query}%"), Sketch.description.ilike(f"%{search_query}%"), )) if not sketches: pagination = filtered_sketches.paginate(page=page, per_page=per_page) sketches = pagination.items has_next = pagination.has_next has_prev = pagination.has_prev next_page = pagination.next_num prev_page = pagination.prev_num current_page = pagination.page total_pages = pagination.pages total_items = pagination.total for sketch in sketches: # Return a subset of the sketch objects to reduce the amount of # data sent to the client. return_sketches.append({ "id": sketch.id, "name": sketch.name, "description": sketch.description, "created_at": str(sketch.created_at), "last_activity": utils.get_sketch_last_activity(sketch), "user": sketch.user.username, "status": sketch.get_status.status, }) meta = { "current_user": current_user.username, "has_next": has_next, "has_prev": has_prev, "next_page": next_page, "prev_page": prev_page, "current_page": current_page, "total_pages": total_pages, "total_items": total_items, } return jsonify({"objects": return_sketches, "meta": meta})
def get(self, sketch_id): """Handles GET request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ if current_user.admin: sketch = Sketch.query.get(sketch_id) if not sketch.has_permission(current_user, "read"): return self._get_sketch_for_admin(sketch) else: sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.") aggregators = {} for _, cls in aggregator_manager.AggregatorManager.get_aggregators(): aggregators[cls.NAME] = { "form_fields": cls.FORM_FIELDS, "display_name": cls.DISPLAY_NAME, "description": cls.DESCRIPTION, } # Get mappings for all indices in the sketch. This is used to set # columns shown in the event list. sketch_indices = [ t.searchindex.index_name for t in sketch.active_timelines ] # Make sure the list of index names is uniq sketch_indices = list(set(sketch_indices)) # Get event count and size on disk for each index in the sketch. indices_metadata = {} stats_per_timeline = {} for timeline in sketch.active_timelines: indices_metadata[timeline.searchindex.index_name] = {} stats_per_timeline[timeline.id] = {"count": 0} if not sketch_indices: mappings_settings = {} else: try: mappings_settings = self.datastore.client.indices.get_mapping( index=sketch_indices) except opensearchpy.NotFoundError: logger.error("Unable to get indices mapping in datastore, for " "indices: {0:s}".format(",".join(sketch_indices))) mappings_settings = {} mappings = [] for index_name, value in mappings_settings.items(): # The structure is different in ES version 6.x and lower. This check # makes sure we support both old and new versions. properties = value["mappings"].get("properties") if not properties: properties = next(iter( value["mappings"].values())).get("properties") # Determine if index is from the time before multiple timelines per # index. This is used in the UI to support both modes. is_legacy = bool("__ts_timeline_id" not in properties) indices_metadata[index_name]["is_legacy"] = is_legacy for field, value_dict in properties.items(): mapping_dict = {} # Exclude internal fields if field.startswith("__"): continue if field == "timesketch_label": continue mapping_dict["field"] = field mapping_dict["type"] = value_dict.get("type", "n/a") mappings.append(mapping_dict) # Get number of events per timeline if sketch_indices: # Support legacy indices. for timeline in sketch.active_timelines: index_name = timeline.searchindex.index_name if indices_metadata[index_name].get("is_legacy", False): doc_count, _ = self.datastore.count(indices=index_name) stats_per_timeline[timeline.id] = {"count": doc_count} count_agg_spec = { "aggs": { "per_timeline": { "terms": { "field": "__ts_timeline_id", "size": len(sketch.timelines), } } } } # pylint: disable=unexpected-keyword-arg, no-value-for-parameter count_agg = self.datastore.client.search(index=sketch_indices, body=count_agg_spec, size=0) count_per_timeline = (count_agg.get("aggregations", {}).get("per_timeline", {}).get("buckets", [])) for count_stat in count_per_timeline: stats_per_timeline[count_stat["key"]] = { "count": count_stat["doc_count"] } # Make the list of dicts unique mappings = {v["field"]: v for v in mappings}.values() views = [] for view in sketch.get_named_views: if not view.user: username = "******" else: username = view.user.username view = { "name": view.name, "description": view.description, "id": view.id, "query": view.query_string, "filter": view.query_filter, "user": username, "created_at": view.created_at, "updated_at": view.updated_at, } views.append(view) views.sort(key=lambda x: x.get("name", "Z").lower()) stories = [] for story in sketch.stories: if not story.user: username = "******" else: username = story.user.username story = { "id": story.id, "title": story.title, "user": username, "created_at": story.created_at, "updated_at": story.updated_at, } stories.append(story) meta = dict( aggregators=aggregators, views=views, stories=stories, searchtemplates=[{ "name": searchtemplate.name, "id": searchtemplate.id } for searchtemplate in SearchTemplate.query.all()], emojis=get_emojis_as_dict(), permissions={ "public": bool(sketch.is_public), "read": bool(sketch.has_permission(current_user, "read")), "write": bool(sketch.has_permission(current_user, "write")), "delete": bool(sketch.has_permission(current_user, "delete")), }, collaborators={ "users": [user.username for user in sketch.collaborators], "groups": [group.name for group in sketch.groups], }, attributes=utils.get_sketch_attributes(sketch), mappings=list(mappings), indices_metadata=indices_metadata, stats_per_timeline=stats_per_timeline, last_activity=utils.get_sketch_last_activity(sketch), filter_labels=self.datastore.get_filter_labels( sketch.id, sketch_indices), sketch_labels=[label.label for label in sketch.labels], ) return self.to_json(sketch, meta=meta)