def _kibana_export_obj(self, dest, _type, body):
i = 0
dest = os.path.join(dest, _type)
os.makedirs(dest)
while True:
if get_es_major_version() < 6:
res = self.client.search(index='.kibana', from_=i, doc_type=_type, body=body)
else:
res = self.client.search(index='.kibana', from_=i, body=body)
if len(res['hits']['hits']) == 0:
break
i += 10
for hit in res['hits']['hits']:
_id = hit['_id']
filename = os.path.join(dest, _id)
filename += '.json'
if get_es_major_version() < 6:
res = self.client.get(index='.kibana', doc_type=_type, id=_id)
else:
res = self.client.get(index='.kibana', doc_type='doc', id=_id)
with open(filename, 'w') as f:
f.write(json.dumps(res['_source'], separators= (',', ':')))
def _kibana_remove(self, _type, body):
i = 0
ids = []
if get_es_major_version() >= 6:
body['query']['query_string']['query'] += ' type:%s' % _type
_type = 'doc'
while True:
res = self.client.search(index='.kibana',
from_=i,
doc_type=_type,
body=body,
request_cache=False)
if len(res['hits']['hits']) == 0:
break
i += 10
_ids = [hit['_id'] for hit in res['hits']['hits']]
ids += _ids
for _id in ids:
self.client.delete(index='.kibana',
doc_type=_type,
id=_id,
refresh=True)
def kibana_clear(self):
_types = ('search', 'visualization', 'dashboard')
for _type in _types:
if get_es_major_version() >= 6:
query = 'NOT %s.title: SN' % _type
else:
query = 'NOT title: SN'
body = {'query': {'query_string': {'query': query}}}
self._kibana_remove(_type, body)
def _kibana_inject(self, _type, _file):
with open(_file) as f:
content = f.read()
name = _file.rsplit('/', 1)[1]
name = name.rsplit('.', 1)[0]
if get_es_major_version() < 6:
doc_type = _type
else:
doc_type = 'doc'
self.client.create(index='.kibana', doc_type=doc_type, id=name, body=content, refresh=True)
def _kibana_set_default_index(self, idx):
if get_es_major_version() < 6:
res = self.client.search(index='.kibana', doc_type='config', body={'query': {'match_all': {}}}, request_cache=False)
else:
body = {'query': {'query_string': {'query': 'type: config'}}}
res = self.client.search(index='.kibana', doc_type='doc', body=body, request_cache=False)
for hit in res['hits']['hits']:
content = hit['_source']
content['defaultIndex'] = idx
if get_es_major_version() < 6:
self.client.update(index='.kibana', doc_type='config', id=hit['_id'], body={'doc': content}, refresh=True)
else:
self.client.update(index='.kibana', doc_type='doc', id=hit['_id'], body=content, refresh=True)
else:
if get_es_major_version() >= 6:
self._kibana_request('/api/kibana/settings/defaultIndex', {'value': 'logstash-*'})
else:
print >> sys.stderr, "Warning: unknown ES version, not setting Kibana's defaultIndex"
def get_paginated_response(self, data, full=False):
from rules.es_graphs import get_es_major_version
total = data['hits']['total']
if get_es_major_version() >= 7:
total = total['value']
return Response(OrderedDict([
('count', total),
('next', self.get_next_link(total)),
('previous', self.get_previous_link()),
('results', [entry['_source'] if not full else entry for entry in data['hits']['hits']])
]))
def kibana_export(self, full=False):
dest = tempfile.mkdtemp()
_types = ('search', 'visualization', 'dashboard')
if full:
_types = _types + ('index-pattern', )
for _type in _types:
if get_es_major_version() < 6:
if full:
body = {'query': {'match_all': {}}}
else:
body = {
'query': {
'query_string': {
'query': 'NOT title: SN *'
}
}
}
else:
if full:
body = {
'query': {
'query_string': {
'query': 'type: %s' % _type
}
}
}
else:
body = {
'query': {
'query_string': {
'query':
'type: %s AND NOT %s.title: SN' %
(_type, _type)
}
}
}
self._kibana_export_obj(dest, _type, body)
f = tempfile.NamedTemporaryFile(delete=False)
tar_name = 'scirius-dashboards-%s' % strftime('%Y%m%d%H%M')
tar = tarfile.open(mode='w:bz2', fileobj=f)
tar.add(dest, tar_name)
tar.close()
rmtree(dest)
f.close()
tar_name += '.tar.bz2'
return tar_name, f.name
def _kibana_inject(self, _type, _file):
with open(_file) as f:
content = f.read()
name = _file.rsplit('/', 1)[1]
name = name.rsplit('.', 1)[0]
if get_es_major_version() < 6:
doc_type = _type
else:
doc_type = 'doc'
try:
# Delete the document first, to prevent an error when it's already there
self.client.delete(index='.kibana', doc_type=doc_type, id=name, refresh=True)
except NotFoundError:
pass
self.client.create(index='.kibana', doc_type=doc_type, id=name, body=content, refresh=True)
def kibana_reset(self):
self._create_kibana_mappings()
if not os.path.isdir(self._get_dashboard_dir()):
raise Exception(
'Please make sure Kibana dashboards are installed at %s' %
self._get_dashboard_dir())
if self._get_kibana_subdirfiles('index-pattern') == []:
raise Exception(
'Please make sure Kibana dashboards are installed at %s: no index-pattern found'
% self._get_dashboard_dir())
self._kibana_remove('dashboard',
{'query': {
'query_string': {
'query': 'SN*'
}
}})
self._kibana_remove('visualization',
{'query': {
'query_string': {
'query': 'SN*'
}
}})
self._kibana_remove('search',
{'query': {
'query_string': {
'query': 'SN*'
}
}})
self._kibana_remove('index-pattern',
{'query': {
'query_string': {
'query': '*'
}
}})
for _type in ('index-pattern', 'search', 'visualization', 'dashboard'):
for _file in self._get_kibana_subdirfiles(_type):
self._kibana_inject(_type, _file)
if get_es_major_version() >= 6:
self._kibana_request('/api/spaces/space', KIBANA6_NAMESPACE)
self._kibana_set_default_index('logstash-*')
def _parse_total_hits(self, resp):
from rules.es_graphs import get_es_major_version
if get_es_major_version() < 7:
return resp['hits']['total']
return resp['hits']['total']['value']
def _es_interval_kw(self):
from rules.es_graphs import get_es_major_version
if get_es_major_version() < 7:
return 'interval'
return 'fixed_interval'
def _get_dashboard_dir(self):
if get_es_major_version() < 6 or not hasattr(settings, 'KIBANA6_DASHBOARDS_PATH'):
# Use KIBANA_DASHBOARDS_PATH when using ES<6 or when KIBANA6_DASHBOARDS_PATH is not set
return settings.KIBANA_DASHBOARDS_PATH
return settings.KIBANA6_DASHBOARDS_PATH
def get_kibana_mappings():
if get_es_major_version() < 6:
return { "dashboard":
{ "properties":
{
"title": { "type": "string" },
"hits": { "type": "integer" },
"description": { "type": "string" },
"panelsJSON": { "type": "string" },
"optionsJSON": { "type": "string" },
"uiStateJSON": { "type": "string" },
"version": { "type": "integer" },
"timeRestore": { "type": "boolean" },
"timeTo": { "type": "string" },
"timeFrom": { "type": "string" },
}
} , "search" : { "properties" :
{
"title": { "type": "string" },
"description": { "type": "string" },
"hits": { "type": "integer" },
"columns": { "type": "string" },
"sort": { "type": "string" },
"version": { "type": "integer" }
}
}, "visualization": { "properties":
{
"title": { "type": "string" },
"uiStateJSON": { "type": "string" },
"description": { "type": "string" },
"savedSearchId": { "type": "string" },
"version": { "type": "integer" }
}
}
}
else:
return {
"doc" : {
"properties" : {
"config" : {
"properties" : {
"buildNum" : {
"type" : "keyword"
},
"defaultIndex" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"telemetry:optIn" : {
"type" : "boolean"
}
}
},
"dashboard" : {
"properties" : {
"description" : {
"type" : "text"
},
"hits" : {
"type" : "integer"
},
"kibanaSavedObjectMeta" : {
"properties" : {
"searchSourceJSON" : {
"type" : "text"
}
}
},
"optionsJSON" : {
"type" : "text"
},
"panelsJSON" : {
"type" : "text"
},
"refreshInterval" : {
"properties" : {
"display" : {
"type" : "keyword"
},
"pause" : {
"type" : "boolean"
},
"section" : {
"type" : "integer"
},
"value" : {
"type" : "integer"
}
}
},
"timeFrom" : {
"type" : "keyword"
},
"timeRestore" : {
"type" : "boolean"
},
"timeTo" : {
"type" : "keyword"
},
"title" : {
"type" : "text"
},
"uiStateJSON" : {
"type" : "text"
},
"version" : {
"type" : "integer"
}
}
},
"graph-workspace" : {
"properties" : {
"description" : {
"type" : "text"
},
"kibanaSavedObjectMeta" : {
"properties" : {
"searchSourceJSON" : {
"type" : "text"
}
}
},
"numLinks" : {
"type" : "integer"
},
"numVertices" : {
"type" : "integer"
},
"title" : {
"type" : "text"
},
"version" : {
"type" : "integer"
},
"wsState" : {
"type" : "text"
}
}
},
"index-pattern" : {
"properties" : {
"fieldFormatMap" : {
"type" : "text"
},
"fields" : {
"type" : "text"
},
"intervalName" : {
"type" : "keyword"
},
"notExpandable" : {
"type" : "boolean"
},
"sourceFilters" : {
"type" : "text"
},
"timeFieldName" : {
"type" : "keyword"
},
"title" : {
"type" : "text"
}
}
},
"search" : {
"properties" : {
"columns" : {
"type" : "keyword"
},
"description" : {
"type" : "text"
},
"hits" : {
"type" : "integer"
},
"kibanaSavedObjectMeta" : {
"properties" : {
"searchSourceJSON" : {
"type" : "text"
}
}
},
"sort" : {
"type" : "keyword"
},
"title" : {
"type" : "text"
},
"version" : {
"type" : "integer"
}
}
},
"server" : {
"properties" : {
"uuid" : {
"type" : "keyword"
}
}
},
"timelion-sheet" : {
"properties" : {
"description" : {
"type" : "text"
},
"hits" : {
"type" : "integer"
},
"kibanaSavedObjectMeta" : {
"properties" : {
"searchSourceJSON" : {
"type" : "text"
}
}
},
"timelion_chart_height" : {
"type" : "integer"
},
"timelion_columns" : {
"type" : "integer"
},
"timelion_interval" : {
"type" : "keyword"
},
"timelion_other_interval" : {
"type" : "keyword"
},
"timelion_rows" : {
"type" : "integer"
},
"timelion_sheet" : {
"type" : "text"
},
"title" : {
"type" : "text"
},
"version" : {
"type" : "integer"
}
}
},
"type" : {
"type" : "keyword"
},
"updated_at" : {
"type" : "date"
},
"url" : {
"properties" : {
"accessCount" : {
"type" : "long"
},
"accessDate" : {
"type" : "date"
},
"createDate" : {
"type" : "date"
},
"url" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 2048
}
}
}
}
},
"visualization" : {
"properties" : {
"description" : {
"type" : "text"
},
"kibanaSavedObjectMeta" : {
"properties" : {
"searchSourceJSON" : {
"type" : "text"
}
}
},
"savedSearchId" : {
"type" : "keyword"
},
"title" : {
"type" : "text"
},
"uiStateJSON" : {
"type" : "text"
},
"version" : {
"type" : "integer"
},
"visState" : {
"type" : "text"
}
}
}
}
}
}
