def _export_tagged_events(self, sketch, zip_file): """Export all events that have been tagged and store in a ZIP file.""" result = self.datastore.search(sketch_id=sketch.id, query_string='_exists_:tag', query_filter=self.DEFAULT_QUERY_FILTER, query_dsl='', indices=self.sketch_indices) fh = export.query_results_to_filehandle(result, sketch) zip_file.writestr('events/tagged_events.csv', data=fh.read()) parameters = { 'limit': 100, 'field': 'tag', } result_obj, meta = utils.run_aggregator( sketch.id, aggregator_name='field_bucket', aggregator_parameters=parameters) zip_file.writestr('events/tagged_event_stats.meta', data=json.dumps(meta)) html = result_obj.to_chart(chart_name='hbarchart', chart_title='Top 100 identified tags', interactive=True, as_html=True) zip_file.writestr('events/tagged_event_stats.html', data=html) string_io = io.StringIO() data_frame = result_obj.to_pandas() data_frame.to_csv(string_io, index=False) string_io.seek(0) zip_file.writestr('events/tagged_event_stats.csv', data=string_io.read())
def _export_view(self, view, sketch, zip_file): """Export a view from a sketch and write it to a ZIP file. Args: view (timesketch.models.sketch.View): a View object. sketch (timesketch.models.sketch.Sketch): a sketch object. zip_file (ZipFile): a zip file handle that can be used to write content to. """ name = "{0:04d}_{1:s}".format(view.id, view.name) query_filter = json.loads(view.query_filter) if not query_filter: query_filter = self.DEFAULT_QUERY_FILTER indices = query_filter.get("indices", self.sketch_indices) if not indices or "_all" in indices: indices = self.sketch_indices # Ignoring the size limits in views to reduce the amount of queries # needed to get all the data. query_filter["terminate_after"] = 10000 query_filter["size"] = 10000 query_dsl = view.query_dsl if query_dsl: query_dict = json.loads(query_dsl) if not query_dict: query_dsl = None result = self.datastore.search( sketch_id=sketch.id, query_string=view.query_string, query_filter=query_filter, query_dsl=query_dsl, enable_scroll=True, indices=indices, ) scroll_id = result.get("_scroll_id", "") if scroll_id: data_frame = export.query_results_to_dataframe(result, sketch) total_count = result.get("hits", {}).get("total", {}).get("value", 0) if isinstance(total_count, str): try: total_count = int(total_count, 10) except ValueError: total_count = 0 event_count = len(result["hits"]["hits"]) while event_count < total_count: # pylint: disable=unexpected-keyword-arg result = self.datastore.client.scroll(scroll_id=scroll_id, scroll="1m") event_count += len(result["hits"]["hits"]) add_frame = export.query_results_to_dataframe(result, sketch) if add_frame.shape[0]: data_frame = pd.concat([data_frame, add_frame], sort=False) else: logger.warning( "Data Frame returned from a search operation was " "empty, count {0:d} out of {1:d} total. Query is: " '"{2:s}"'.format( event_count, total_count, view.query_string or query_dsl ) ) fh = io.StringIO() data_frame.to_csv(fh, index=False) fh.seek(0) else: fh = export.query_results_to_filehandle(result, sketch) zip_file.writestr("views/{0:s}.csv".format(name), data=fh.read()) if not view.user: username = "******" else: username = view.user.username meta = { "name": view.name, "view_id": view.id, "description": view.description, "query_string": view.query_string, "query_filter": view.query_filter, "query_dsl": view.query_dsl, "username": username, "sketch_id": view.sketch_id, } zip_file.writestr("views/{0:s}.meta".format(name), data=json.dumps(meta))
def _export_view(self, view, sketch, zip_file): """Export a view from a sketch and write it to a ZIP file. Args: view (timesketch.models.sketch.View): a View object. sketch (timesketch.models.sketch.Sketch): a sketch object. zip_file (ZipFile): a zip file handle that can be used to write content to. """ name = '{0:04d}_{1:s}'.format(view.id, view.name) query_filter = json.loads(view.query_filter) if not query_filter: query_filter = self.DEFAULT_QUERY_FILTER indices = query_filter.get('indices', self.sketch_indices) if not indices or '_all' in indices: indices = self.sketch_indices # Ignoring the size limits in views to reduce the amount of queries # needed to get all the data. query_filter['terminate_after'] = 10000 query_filter['size'] = 10000 query_dsl = view.query_dsl if query_dsl: query_dict = json.loads(query_dsl) if not query_dict: query_dsl = None result = self.datastore.search( sketch_id=sketch.id, query_string=view.query_string, query_filter=query_filter, query_dsl=query_dsl, enable_scroll=True, indices=indices) scroll_id = result.get('_scroll_id', '') if scroll_id: data_frame = export.query_results_to_dataframe(result, sketch) total_count = result.get( 'hits', {}).get('total', {}).get('value', 0) if isinstance(total_count, str): try: total_count = int(total_count, 10) except ValueError: total_count = 0 event_count = len(result['hits']['hits']) while event_count < total_count: # pylint: disable=unexpected-keyword-arg result = self.datastore.client.scroll( scroll_id=scroll_id, scroll='1m') event_count += len(result['hits']['hits']) add_frame = export.query_results_to_dataframe(result, sketch) if add_frame.shape[0]: data_frame = pd.concat([data_frame, add_frame], sort=False) else: logger.warning( 'Data Frame returned from a search operation was ' 'empty, count {0:d} out of {1:d} total. Query is: ' '"{2:s}"'.format( event_count, total_count, view.query_string or query_dsl)) fh = io.StringIO() data_frame.to_csv(fh, index=False) fh.seek(0) else: fh = export.query_results_to_filehandle(result, sketch) zip_file.writestr( 'views/{0:s}.csv'.format(name), data=fh.read()) if not view.user: username = '******' else: username = view.user.username meta = { 'name': view.name, 'view_id': view.id, 'description': view.description, 'query_string': view.query_string, 'query_filter': view.query_filter, 'query_dsl': view.query_dsl, 'username': username, 'sketch_id': view.sketch_id, } zip_file.writestr( 'views/{0:s}.meta'.format(name), data=json.dumps(meta))