def event_stream(sketch_id, query):
es = ElasticsearchDataStore(host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
sketch = Sketch.query.get(sketch_id)
if not sketch:
sys.exit('No such sketch')
indices = {t.searchindex.index_name for t in sketch.timelines}
result = es.search(sketch_id=sketch_id,
query_string=query,
query_filter={u'limit': 10000},
query_dsl={},
indices=[u'_all'],
return_fields=[u'xml_string', u'timestamp'],
enable_scroll=True)
scroll_id = result[u'_scroll_id']
scroll_size = result[u'hits'][u'total']
for event in result[u'hits'][u'hits']:
yield event
while scroll_size > 0:
result = es.client.scroll(scroll_id=scroll_id, scroll=u'1m')
scroll_id = result[u'_scroll_id']
scroll_size = len(result[u'hits'][u'hits'])
for event in result[u'hits'][u'hits']:
yield event
def event_stream(sketch_id, query):
es = ElasticsearchDataStore(
host=current_app.config['ELASTIC_HOST'],
port=current_app.config['ELASTIC_PORT'])
sketch = Sketch.query.get(sketch_id)
if not sketch:
sys.exit('No such sketch')
indices = {t.searchindex.index_name for t in sketch.timelines}
result = es.search(
sketch_id=sketch_id,
query_string=query,
query_filter={'size': 10000, 'terminate_after': 1000},
query_dsl={},
indices=['_all'],
return_fields=['xml_string', 'timestamp'],
enable_scroll=True)
scroll_id = result['_scroll_id']
scroll_size = result['hits']['total']
for event in result['hits']['hits']:
yield event
while scroll_size > 0:
result = es.client.scroll(scroll_id=scroll_id, scroll='1m')
scroll_id = result['_scroll_id']
scroll_size = len(result['hits']['hits'])
for event in result['hits']['hits']:
yield event
def export(sketch_id):
"""Generates CSV from search result.
Args:
sketch_id: Primary key for a sketch.
Returns:
CSV string with header.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
view = sketch.get_user_view(current_user)
query_filter = json.loads(view.query_filter)
query_dsl = json.loads(view.query_dsl)
indices = query_filter.get(u'indices', [])
datastore = ElasticsearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
result = datastore.search(
sketch_id, view.query_string, query_filter, query_dsl, indices,
aggregations=None, return_results=True)
csv_out = StringIO()
csv_writer = csv.DictWriter(
csv_out, fieldnames=[
u'timestamp', u'message', u'timestamp_desc', u'datetime',
u'timesketch_label', u'tag'])
csv_writer.writeheader()
for _event in result[u'hits'][u'hits']:
csv_writer.writerow(
dict((k, v.encode(u'utf-8') if isinstance(v, basestring) else v)
for k, v in _event[u'_source'].iteritems()))
return csv_out.getvalue()
def export(sketch_id):
"""Generates CSV from search result.
Args:
sketch_id: Primary key for a sketch.
Returns:
CSV string with header.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
view = sketch.get_user_view(current_user)
query_filter = json.loads(view.query_filter)
query_dsl = json.loads(view.query_dsl)
indices = query_filter.get('indices', [])
# Export more than the 500 first results.
max_events_to_fetch = 10000
query_filter['terminate_after'] = max_events_to_fetch
query_filter['size'] = max_events_to_fetch
datastore = ElasticsearchDataStore(
host=current_app.config['ELASTIC_HOST'],
port=current_app.config['ELASTIC_PORT'])
result = datastore.search(
sketch_id,
view.query_string,
query_filter,
query_dsl,
indices,
aggregations=None)
all_fields = set()
for event in result['hits']['hits']:
all_fields.update(event['_source'].keys())
all_fields.difference_update(DEFAULT_FIELDS)
fieldnames = DEFAULT_FIELDS + sorted(all_fields)
csv_out = StringIO()
csv_writer = csv.DictWriter(csv_out, fieldnames=fieldnames)
csv_writer.writeheader()
for _event in result['hits']['hits']:
sources = _event['_source']
row = {}
for key, value in iter(sources.items()):
if isinstance(value, six.binary_type):
value = codecs.decode(value, 'utf-8')
row[key] = value
row['_index'] = _event['_index']
if isinstance(row['_index'], six.binary_type):
row['_index'] = row['_index'].encode('utf-8')
csv_writer.writerow(row)
return csv_out.getvalue()
def export(sketch_id):
"""Generates CSV from search result.
Args:
sketch_id: Primary key for a sketch.
Returns:
CSV string with header.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
view = sketch.get_user_view(current_user)
query_filter = json.loads(view.query_filter)
query_dsl = json.loads(view.query_dsl)
indices = query_filter.get('indices', [])
# Export more than the 500 first results.
max_events_to_fetch = 10000
query_filter['terminate_after'] = max_events_to_fetch
query_filter['size'] = max_events_to_fetch
datastore = ElasticsearchDataStore(
host=current_app.config['ELASTIC_HOST'],
port=current_app.config['ELASTIC_PORT'])
result = datastore.search(
sketch_id,
view.query_string,
query_filter,
query_dsl,
indices,
aggregations=None)
all_fields = set()
for event in result['hits']['hits']:
all_fields.update(event['_source'].keys())
all_fields.difference_update(DEFAULT_FIELDS)
fieldnames = DEFAULT_FIELDS + sorted(all_fields)
csv_out = StringIO()
csv_writer = csv.DictWriter(csv_out, fieldnames=fieldnames)
csv_writer.writeheader()
for _event in result['hits']['hits']:
sources = _event['_source']
row = {}
for key, value in iter(sources.items()):
if isinstance(value, six.binary_type):
value = codecs.decode(value, 'utf-8')
row[key] = value
row['_index'] = _event['_index']
if isinstance(row['_index'], six.binary_type):
row['_index'] = row['_index'].encode('utf-8')
csv_writer.writerow(row)
return csv_out.getvalue()
def export(sketch_id):
"""Generates CSV from search result.
Args:
sketch_id: Primary key for a sketch.
Returns:
CSV string with header.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
view = sketch.get_user_view(current_user)
query_filter = json.loads(view.query_filter)
query_dsl = json.loads(view.query_dsl)
indices = query_filter.get(u'indices', [])
# Export more than the 500 first results.
max_events_to_fetch = 10000
query_filter[u'limit'] = max_events_to_fetch
datastore = ElasticsearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
result = datastore.search(
sketch_id,
view.query_string,
query_filter,
query_dsl,
indices,
aggregations=None)
all_fields = set()
for event in result[u'hits'][u'hits']:
all_fields.update(event[u'_source'].keys())
all_fields.difference_update(DEFAULT_FIELDS)
fieldnames = DEFAULT_FIELDS + sorted(all_fields)
csv_out = StringIO()
csv_writer = csv.DictWriter(csv_out, fieldnames=fieldnames)
csv_writer.writeheader()
for _event in result[u'hits'][u'hits']:
row = dict((k, v.encode(u'utf-8') if isinstance(v, basestring) else v)
for k, v in _event[u'_source'].iteritems())
row[u'_index'] = _event[u'_index']
if isinstance(row[u'_index'], basestring):
row[u'_index'] = row[u'_index'].encode(u'utf-8')
csv_writer.writerow(row)
return csv_out.getvalue()
