def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', help='Elasticsearch host')
parser.add_argument('--port', type=int, help='Elasticsearch port')
parser.add_argument('--url-prefix', help='Elasticsearch URL prefix')
parser.add_argument('--no-auth', action='store_const', const=True, help='Suppress prompt for basic auth')
parser.add_argument('--ssl', action='store_true', default=None, help='Use SSL')
parser.add_argument('--no-ssl', dest='ssl', action='store_false', help='Do not use SSL')
parser.add_argument('--index', help='Index name to create')
parser.add_argument('--old-index', help='Old index name to copy')
args = parser.parse_args()
if os.path.isfile('../config.yaml'):
filename = '../config.yaml'
elif os.path.isfile('config.yaml'):
filename = 'config.yaml'
else:
filename = ''
username = None
password = None
use_ssl = None
url_prefix = None
http_auth = None
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = data.get('es_host')
port = data.get('es_port')
username = data.get('es_username')
password = data.get('es_password')
url_prefix = data.get('es_url_prefix', '')
use_ssl = data.get('use_ssl')
else:
host = args.host if args.host else raw_input('Enter elasticsearch host: ')
port = args.port if args.port else int(raw_input('Enter elasticsearch port: '))
use_ssl = (args.ssl if args.ssl is not None
else raw_input('Use SSL? t/f: ').lower() in ('t', 'true'))
if args.no_auth is None:
username = raw_input('Enter optional basic-auth username: '******'Enter optional basic-auth password: '******'Enter optional Elasticsearch URL prefix: '))
if username and password:
http_auth = username + ':' + password
es = Elasticsearch(host=host, port=port, use_ssl=use_ssl, http_auth=http_auth, url_prefix=url_prefix)
silence_mapping = {'silence': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'until': {'type': 'date', 'format': 'dateOptionalTime'},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'}}}}
ess_mapping = {'elastalert_status': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'}}}}
es_mapping = {'elastalert': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'},
'match_body': {'enabled': False, 'type': 'object'},
'aggregate_id': {'index': 'not_analyzed', 'type': 'string'}}}}
past_mapping = {'past_elastalert': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'match_body': {'enabled': False, 'type': 'object'},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'},
'aggregate_id': {'index': 'not_analyzed', 'type': 'string'}}}}
error_mapping = {'elastalert_error': {'properties': {'data': {'type': 'object', 'enabled': False},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'}}}}
index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ')
if not index:
index = 'elastalert_status'
old_index = (args.old_index if args.old_index is not None
else raw_input('Name of existing index to copy? (Default None) '))
res = None
if old_index:
print('Downloading existing data...')
res = es.search(index=old_index, body={}, size=500000)
print('Got %s documents' % (len(res['hits']['hits'])))
es.indices.create(index)
# To avoid a race condition. TODO: replace this with a real check
time.sleep(2)
es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_status', body=ess_mapping)
es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping)
es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping)
print('New index %s created' % (index))
if res:
bulk = ''.join(['%s\n%s\n' % (json.dumps({'create': {'_type': doc['_type'], '_index': index}}),
json.dumps(doc['_source'])) for doc in res['hits']['hits']])
print('Uploading data...')
es.bulk(body=bulk, index=index)
print('Done!')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', help='Elasticsearch host')
parser.add_argument('--port', type=int, help='Elasticsearch port')
parser.add_argument('--url-prefix', help='Elasticsearch URL prefix')
parser.add_argument('--no-auth',
action='store_const',
const=True,
help='Suppress prompt for basic auth')
parser.add_argument('--ssl',
action='store_true',
default=None,
help='Use SSL')
parser.add_argument('--no-ssl',
dest='ssl',
action='store_false',
help='Do not use SSL')
parser.add_argument('--index', help='Index name to create')
parser.add_argument('--old-index', help='Old index name to copy')
parser.add_argument('--boto-profile',
default=None,
help='Boto profile to use for signing requests')
parser.add_argument('--aws-region',
default=None,
help='AWS Region to use for signing requests')
args = parser.parse_args()
if os.path.isfile('../config.yaml'):
filename = '../config.yaml'
elif os.path.isfile('config.yaml'):
filename = 'config.yaml'
else:
filename = ''
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = args.host if args.host else data.get('es_host')
port = args.port if args.port else data.get('es_port')
username = data.get('es_username')
password = data.get('es_password')
url_prefix = args.url_prefix if args.url_prefix is not None else data.get(
'es_url_prefix', '')
use_ssl = args.ssl if args.ssl is not None else data.get('use_ssl')
aws_region = data.get('aws_region', None)
else:
username = None
password = None
aws_region = args.aws_region
host = args.host if args.host else raw_input(
'Enter elasticsearch host: ')
port = args.port if args.port else int(
raw_input('Enter elasticsearch port: '))
use_ssl = (args.ssl if args.ssl is not None else
raw_input('Use SSL? t/f: ').lower() in ('t', 'true'))
if args.no_auth is None:
username = raw_input('Enter optional basic-auth username: '******'Enter optional basic-auth password: '******'Enter optional Elasticsearch URL prefix: '))
auth = Auth()
http_auth = auth(host=host,
username=username,
password=password,
aws_region=aws_region,
boto_profile=args.boto_profile)
es = Elasticsearch(host=host,
port=port,
use_ssl=use_ssl,
connection_class=RequestsHttpConnection,
http_auth=http_auth,
url_prefix=url_prefix)
silence_mapping = {
'silence': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'until': {
'type': 'date',
'format': 'dateOptionalTime'
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
}
ess_mapping = {
'elastalert_status': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
}
es_mapping = {
'elastalert': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
},
'alert_time': {
'format': 'dateOptionalTime',
'type': 'date'
},
'match_body': {
'enabled': False,
'type': 'object'
},
'aggregate_id': {
'index': 'not_analyzed',
'type': 'string'
}
}
}
}
past_mapping = {
'past_elastalert': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'match_body': {
'enabled': False,
'type': 'object'
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
},
'aggregate_id': {
'index': 'not_analyzed',
'type': 'string'
}
}
}
}
error_mapping = {
'elastalert_error': {
'properties': {
'data': {
'type': 'object',
'enabled': False
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
}
index = args.index if args.index is not None else raw_input(
'New index name? (Default elastalert_status) ')
if not index:
index = 'elastalert_status'
old_index = (args.old_index if args.old_index is not None else
raw_input('Name of existing index to copy? (Default None) '))
res = None
if old_index:
print('Downloading existing data...')
res = es.search(index=old_index, body={}, size=500000)
print('Got %s documents' % (len(res['hits']['hits'])))
es_index = IndicesClient(es)
if es_index.exists(index):
print('Index ' + index + ' already exists. Skipping index creation.')
return None
es.indices.create(index)
# To avoid a race condition. TODO: replace this with a real check
time.sleep(2)
es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping)
es.indices.put_mapping(index=index,
doc_type='elastalert_status',
body=ess_mapping)
es.indices.put_mapping(index=index,
doc_type='silence',
body=silence_mapping)
es.indices.put_mapping(index=index,
doc_type='elastalert_error',
body=error_mapping)
es.indices.put_mapping(index=index,
doc_type='past_elastalert',
body=past_mapping)
print('New index %s created' % index)
if res:
bulk = ''.join([
'%s\n%s\n' %
(json.dumps({'create': {
'_type': doc['_type'],
'_index': index
}}), json.dumps(doc['_source'])) for doc in res['hits']['hits']
])
print('Uploading data...')
es.bulk(body=bulk, index=index)
print('Done!')
def main():
if os.path.isfile('../config.yaml'):
filename = '../config.yaml'
elif os.path.isfile('config.yaml'):
filename = 'config.yaml'
else:
filename = ''
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = data.get('es_host')
port = data.get('es_port')
else:
host = raw_input("Enter elasticsearch host: ")
port = int(raw_input("Enter elasticsearch port: "))
es = Elasticsearch(host=host, port=port)
silence_mapping = {
'silence': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
}
}
}
}
ess_mapping = {
'elastalert_status': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'@timestamp': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
}
es_mapping = {
'elastalert': {
'properties': {
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'match_body': {
'enabled': False,
'type': 'object'
}
}
}
}
error_mapping = {
'elastalert_error': {
'properties': {
'data': {
'type': 'object',
'enabled': False
}
}
}
}
index = raw_input('New index name? (Default elastalert_status) ')
index = index if index else 'elastalert_status'
old_index = raw_input('Name of existing index to copy? (Default None) ')
res = None
if old_index:
print("Downloading existing data...")
res = es.search(index=old_index, body={}, size=500000)
print("Got %s documents" % (len(res['hits']['hits'])))
es.indices.create(index)
es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping)
es.indices.put_mapping(index=index,
doc_type='elastalert_status',
body=ess_mapping)
es.indices.put_mapping(index=index,
doc_type='silence',
body=silence_mapping)
es.indices.put_mapping(index=index,
doc_type='elastalert_error',
body=error_mapping)
print("New index %s created" % (index))
if res:
bulk = ''.join([
'%s\n%s\n' %
(json.dumps({'create': {
'_type': doc['_type'],
'_index': index
}}), json.dumps(doc['_source'])) for doc in res['hits']['hits']
])
print("Uploading data...")
es.bulk(body=bulk, index=index)
print("Done!")
def reindex(from_hosts,
from_index,
to_hosts,
to_index,
to_type,
source='{"query":{"match_all":{}}}',
max_docs=0,
page_size=10,
logging_per_docs=1000,
es_scroll='5m',
request_timeout=60):
if from_index is None:
logger.warn('from_index is empty.')
return
from_es = Elasticsearch(hosts=from_hosts)
to_es = Elasticsearch(hosts=to_hosts)
scroll_id = None
counter = 0
running = True
bulk_data = []
while(running):
try:
if scroll_id is None:
response = from_es.search(index=from_index,
body=source,
params={"request_timeout": request_timeout,
"scroll": es_scroll,
"size": page_size})
else:
response = from_es.scroll(scroll_id=scroll_id,
params={"request_timeout": request_timeout,
"scroll": es_scroll})
if len(response['hits']['hits']) == 0:
running = False
break
scroll_id = response['_scroll_id']
for hit in response['hits']['hits']:
if '_source' in hit:
counter += 1
if counter % logging_per_docs == 0:
logger.info(u'Loaded {0} docs.'.format(counter))
if max_docs > 0 and counter >= max_docs:
logger.info(u'{0} docs are loaded, but it exceeded {1} docs.'.format(counter, max_docs))
running = False
break
op_index = to_index if to_index is not None else hit['_index']
op_type = to_type if to_type is not None else hit['_type']
bulk_data.append({"index": {"_index": op_index,
"_type": op_type,
"_id": hit['_id']}
})
bulk_data.append(hit['_source'])
if len(bulk_data) != 0:
to_es.bulk(body=bulk_data, params={"request_timeout": request_timeout})
bulk_data = []
except NotFoundError:
break
except:
logger.exception(u"Failed to load documents from Elasticsearch(Loaded {0} doc).".format(counter))
break
if len(bulk_data) != 0:
to_es.bulk(body=bulk_data, params={"request_timeout": request_timeout})
logger.info('Loaded {0} documents.'.format(counter))
class BaseElasticsearchBackend(Base):
"""Base connection wrapper based on the ElasticSearch official library.
It uses two entry points to configure the underlying connection:
* ``transport_class``: the transport class from ``elasticsearch``. By
default ``elasticsearch.transport.Transport``.
* ``connection_class``: the connection class used by the transport class.
It's undefined by default, as it is on the subclasses to provide one.
If any of these elements is not defined, an ``ImproperlyConfigured`` error
will be raised when the backend will try to configure the client.
"""
#: ElasticSearch transport class used by the client class to perform
#: requests.
transport_class = Transport
#: ElasticSearch connection class used by the transport class to perform
#: requests.
connection_class = None
def configure_client(self):
"""Instantiate and configure the ElasticSearch client.
It simply takes the given HOSTS list and uses PARAMS as the keyword
arguments of the ElasticSearch class.
The client's transport_class is given by the class attribute
``transport_class``, and the connection class used by the transport
class is given by the class attribute ``connection_class``.
An ``ImproperlyConfigured`` exception is raised if any of these
elements is undefined.
"""
hosts = self.server['HOSTS']
params = self.server['PARAMS']
if not self.transport_class:
raise ImproperlyConfigured(
'Djangoes backend %r is not properly configured: '
'no transport class provided' % self.__class__)
if not self.connection_class:
raise ImproperlyConfigured(
'Djangoes backend %r is not properly configured: '
'no connection class provided' % self.__class__)
#pylint: disable=star-args
self.client = Elasticsearch(hosts,
transport_class=self.transport_class,
connection_class=self.connection_class,
**params)
# Server methods
# ==============
# The underlying client does not require index names to perform server
# related queries, such as "ping" or "info". The connection wrapper act
# for them as a proxy.
def ping(self, **kwargs):
return self.client.ping(**kwargs)
def info(self, **kwargs):
return self.client.info(**kwargs)
def put_script(self, lang, script_id, body, **kwargs):
return self.client.put_script(lang, script_id, body, **kwargs)
def get_script(self, lang, script_id, **kwargs):
return self.client.get_script(lang, script_id, **kwargs)
def delete_script(self, lang, script_id, **kwargs):
return self.client.delete_script(lang, script_id, **kwargs)
def put_template(self, template_id, body, **kwargs):
return self.client.put_template(template_id, body, **kwargs)
def get_template(self, template_id, body=None, **kwargs):
return self.client.get_template(template_id, body, **kwargs)
def delete_template(self, template_id=None, **kwargs):
return self.client.delete_template(template_id, **kwargs)
# Bulk methods
# ============
# The underlying client does not require index names, but it can be used.
# As it makes sense to not give an index, developers are free to use these
# as they want, as long as they are careful.
def mget(self, body, index=None, doc_type=None, **kwargs):
return self.client.mget(body, index, doc_type, **kwargs)
def bulk(self, body, index=None, doc_type=None, **kwargs):
return self.client.bulk(body, index, doc_type, **kwargs)
def msearch(self, body, index=None, doc_type=None, **kwargs):
return self.client.msearch(body, index, doc_type, **kwargs)
def mpercolate(self, body, index=None, doc_type=None, **kwargs):
return self.client.mpercolate(body, index, doc_type, **kwargs)
# Scroll methods
# ==============
# The underlying client does not require an index to perform scroll.
def scroll(self, scroll_id, **kwargs):
return self.client.scroll(scroll_id, **kwargs)
def clear_scroll(self, scroll_id, body=None, **kwargs):
return self.client.clear_scroll(scroll_id, body, **kwargs)
# Query methods
# =============
# The underlying client requires index names (or alias names) to perform
# queries. The connection wrapper overrides these client methods to
# automatically uses the configured names (indices and/or aliases).
def create(self, doc_type, body, doc_id=None, **kwargs):
return self.client.create(
self.indices, doc_type, body, doc_id, **kwargs)
def index(self, doc_type, body, doc_id=None, **kwargs):
return self.client.index(
self.indices, doc_type, body, doc_id, **kwargs)
def exists(self, doc_id, doc_type='_all', **kwargs):
return self.client.exists(self.indices, doc_id, doc_type, **kwargs)
def get(self, doc_id, doc_type='_all', **kwargs):
return self.client.get(self.indices, doc_id, doc_type, **kwargs)
def get_source(self, doc_id, doc_type='_all', **kwargs):
return self.client.get_source(self.indices, doc_id, doc_type, **kwargs)
def update(self, doc_type, doc_id, body=None, **kwargs):
return self.client.update(
self.indices, doc_type, doc_id, body, **kwargs)
def search(self, doc_type=None, body=None, **kwargs):
return self.client.search(self.indices, doc_type, body, **kwargs)
def search_shards(self, doc_type=None, **kwargs):
return self.client.search_shards(self.indices, doc_type, **kwargs)
def search_template(self, doc_type=None, body=None, **kwargs):
return self.client.search_template(
self.indices, doc_type, body, **kwargs)
def explain(self, doc_type, doc_id, body=None, **kwargs):
return self.client.explain(
self.indices, doc_type, doc_id, body, **kwargs)
def delete(self, doc_type, doc_id, **kwargs):
return self.client.delete(self.indices, doc_type, doc_id, **kwargs)
def count(self, doc_type=None, body=None, **kwargs):
return self.client.count(self.indices, doc_type, body, **kwargs)
def delete_by_query(self, doc_type=None, body=None, **kwargs):
return self.client.delete_by_query(
self.indices, doc_type, body, **kwargs)
def suggest(self, body, **kwargs):
return self.client.suggest(body, self.indices, **kwargs)
def percolate(self, doc_type, doc_id=None, body=None, **kwargs):
return self.client.percolate(
self.indices, doc_type, doc_id, body, **kwargs)
def count_percolate(self, doc_type, doc_id=None, body=None, **kwargs):
return self.client.count_percolate(
self.indices, doc_type, doc_id, body, **kwargs)
def mlt(self, doc_type, doc_id, body=None, **kwargs):
return self.client.mlt(self.indices, doc_type, doc_id, body, **kwargs)
def termvector(self, doc_type, doc_id, body=None, **kwargs):
return self.client.termvector(
self.indices, doc_type, doc_id, body, **kwargs)
def mtermvectors(self, doc_type=None, body=None, **kwargs):
return self.client.mtermvectors(self.indices, doc_type, body, **kwargs)
def benchmark(self, doc_type=None, body=None, **kwargs):
return self.client.benchmark(self.indices, doc_type, body, **kwargs)
def abort_benchmark(self, name=None, **kwargs):
return self.client.abort_benchmark(name, **kwargs)
def list_benchmarks(self, doc_type=None, **kwargs):
return self.client.list_benchmarks(self.indices, doc_type, **kwargs)
def main():
if os.path.isfile('../config.yaml'):
filename = '../config.yaml'
elif os.path.isfile('config.yaml'):
filename = 'config.yaml'
else:
filename = ''
username = None
password = None
use_ssl = None
http_auth = None
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = data.get('es_host')
port = data.get('es_port')
username = data.get('es_username')
password = data.get('es_password')
use_ssl = data.get('use_ssl')
else:
host = raw_input("Enter elasticsearch host: ")
port = int(raw_input("Enter elasticsearch port: "))
while use_ssl is None:
resp = raw_input("Use SSL? t/f: ").lower()
use_ssl = True if resp in ('t', 'true') else (False if resp in ('f', 'false') else None)
username = raw_input("Enter optional basic-auth username: "******"Enter optional basic-auth password: "******"Downloading existing data...")
res = es.search(index=old_index, body={}, size=500000)
print("Got %s documents" % (len(res['hits']['hits'])))
es.indices.create(index)
es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_status', body=ess_mapping)
es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping)
print("New index %s created" % (index))
if res:
bulk = ''.join(['%s\n%s\n' % (json.dumps({'create': {'_type': doc['_type'], '_index': index}}),
json.dumps(doc['_source'])) for doc in res['hits']['hits']])
print("Uploading data...")
es.bulk(body=bulk, index=index)
print("Done!")
class BaseElasticsearchBackend(Base):
"""Base connection wrapper based on the ElasticSearch official library.
It uses two entry points to configure the underlying connection:
* ``transport_class``: the transport class from ``elasticsearch``. By
default ``elasticsearch.transport.Transport``.
* ``connection_class``: the connection class used by the transport class.
It's undefined by default, as it is on the subclasses to provide one.
If any of these elements is not defined, an ``ImproperlyConfigured`` error
will be raised when the backend will try to configure the client.
"""
#: ElasticSearch transport class used by the client class to perform
#: requests.
transport_class = Transport
#: ElasticSearch connection class used by the transport class to perform
#: requests.
connection_class = None
def configure_client(self):
"""Instantiate and configure the ElasticSearch client.
It simply takes the given HOSTS list and uses PARAMS as the keyword
arguments of the ElasticSearch class.
The client's transport_class is given by the class attribute
``transport_class``, and the connection class used by the transport
class is given by the class attribute ``connection_class``.
An ``ImproperlyConfigured`` exception is raised if any of these
elements is undefined.
"""
hosts = self.server['HOSTS']
params = self.server['PARAMS']
if not self.transport_class:
raise ImproperlyConfigured(
'Djangoes backend %r is not properly configured: '
'no transport class provided' % self.__class__)
if not self.connection_class:
raise ImproperlyConfigured(
'Djangoes backend %r is not properly configured: '
'no connection class provided' % self.__class__)
#pylint: disable=star-args
self.client = Elasticsearch(hosts,
transport_class=self.transport_class,
connection_class=self.connection_class,
**params)
# Server methods
# ==============
# The underlying client does not require index names to perform server
# related queries, such as "ping" or "info". The connection wrapper act
# for them as a proxy.
def ping(self, **kwargs):
return self.client.ping(**kwargs)
def info(self, **kwargs):
return self.client.info(**kwargs)
def put_script(self, lang, script_id, body, **kwargs):
return self.client.put_script(lang, script_id, body, **kwargs)
def get_script(self, lang, script_id, **kwargs):
return self.client.get_script(lang, script_id, **kwargs)
def delete_script(self, lang, script_id, **kwargs):
return self.client.delete_script(lang, script_id, **kwargs)
def put_template(self, template_id, body, **kwargs):
return self.client.put_template(template_id, body, **kwargs)
def get_template(self, template_id, body=None, **kwargs):
return self.client.get_template(template_id, body, **kwargs)
def delete_template(self, template_id=None, **kwargs):
return self.client.delete_template(template_id, **kwargs)
# Bulk methods
# ============
# The underlying client does not require index names, but it can be used.
# As it makes sense to not give an index, developers are free to use these
# as they want, as long as they are careful.
def mget(self, body, index=None, doc_type=None, **kwargs):
return self.client.mget(body, index, doc_type, **kwargs)
def bulk(self, body, index=None, doc_type=None, **kwargs):
return self.client.bulk(body, index, doc_type, **kwargs)
def msearch(self, body, index=None, doc_type=None, **kwargs):
return self.client.msearch(body, index, doc_type, **kwargs)
def mpercolate(self, body, index=None, doc_type=None, **kwargs):
return self.client.mpercolate(body, index, doc_type, **kwargs)
# Scroll methods
# ==============
# The underlying client does not require an index to perform scroll.
def scroll(self, scroll_id, **kwargs):
return self.client.scroll(scroll_id, **kwargs)
def clear_scroll(self, scroll_id, body=None, **kwargs):
return self.client.clear_scroll(scroll_id, body, **kwargs)
# Query methods
# =============
# The underlying client requires index names (or alias names) to perform
# queries. The connection wrapper overrides these client methods to
# automatically uses the configured names (indices and/or aliases).
def create(self, doc_type, body, doc_id=None, **kwargs):
return self.client.create(self.indices, doc_type, body, doc_id,
**kwargs)
def index(self, doc_type, body, doc_id=None, **kwargs):
return self.client.index(self.indices, doc_type, body, doc_id,
**kwargs)
def exists(self, doc_id, doc_type='_all', **kwargs):
return self.client.exists(self.indices, doc_id, doc_type, **kwargs)
def get(self, doc_id, doc_type='_all', **kwargs):
return self.client.get(self.indices, doc_id, doc_type, **kwargs)
def get_source(self, doc_id, doc_type='_all', **kwargs):
return self.client.get_source(self.indices, doc_id, doc_type, **kwargs)
def update(self, doc_type, doc_id, body=None, **kwargs):
return self.client.update(self.indices, doc_type, doc_id, body,
**kwargs)
def search(self, doc_type=None, body=None, **kwargs):
return self.client.search(self.indices, doc_type, body, **kwargs)
def search_shards(self, doc_type=None, **kwargs):
return self.client.search_shards(self.indices, doc_type, **kwargs)
def search_template(self, doc_type=None, body=None, **kwargs):
return self.client.search_template(self.indices, doc_type, body,
**kwargs)
def explain(self, doc_type, doc_id, body=None, **kwargs):
return self.client.explain(self.indices, doc_type, doc_id, body,
**kwargs)
def delete(self, doc_type, doc_id, **kwargs):
return self.client.delete(self.indices, doc_type, doc_id, **kwargs)
def count(self, doc_type=None, body=None, **kwargs):
return self.client.count(self.indices, doc_type, body, **kwargs)
def delete_by_query(self, doc_type=None, body=None, **kwargs):
return self.client.delete_by_query(self.indices, doc_type, body,
**kwargs)
def suggest(self, body, **kwargs):
return self.client.suggest(body, self.indices, **kwargs)
def percolate(self, doc_type, doc_id=None, body=None, **kwargs):
return self.client.percolate(self.indices, doc_type, doc_id, body,
**kwargs)
def count_percolate(self, doc_type, doc_id=None, body=None, **kwargs):
return self.client.count_percolate(self.indices, doc_type, doc_id,
body, **kwargs)
def mlt(self, doc_type, doc_id, body=None, **kwargs):
return self.client.mlt(self.indices, doc_type, doc_id, body, **kwargs)
def termvector(self, doc_type, doc_id, body=None, **kwargs):
return self.client.termvector(self.indices, doc_type, doc_id, body,
**kwargs)
def mtermvectors(self, doc_type=None, body=None, **kwargs):
return self.client.mtermvectors(self.indices, doc_type, body, **kwargs)
def benchmark(self, doc_type=None, body=None, **kwargs):
return self.client.benchmark(self.indices, doc_type, body, **kwargs)
def abort_benchmark(self, name=None, **kwargs):
return self.client.abort_benchmark(name, **kwargs)
def list_benchmarks(self, doc_type=None, **kwargs):
return self.client.list_benchmarks(self.indices, doc_type, **kwargs)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', help='Elasticsearch host')
parser.add_argument('--port', type=int, help='Elasticsearch port')
parser.add_argument('--url-prefix', help='Elasticsearch URL prefix')
parser.add_argument('--no-auth', action='store_const', const=True, help='Suppress prompt for basic auth')
parser.add_argument('--ssl', action='store_const', const=True, help='Use SSL')
parser.add_argument('--no-ssl', action='store_const', const=True, help='Do not use SSL')
parser.add_argument('--index', help='Index name to create')
parser.add_argument('--old-index', help='Old index name to copy')
args = parser.parse_args()
if os.path.isfile('../config.yaml'):
filename = '../config.yaml'
elif os.path.isfile('config.yaml'):
filename = 'config.yaml'
else:
filename = ''
username = None
password = None
use_ssl = None
url_prefix = None
http_auth = None
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = data.get('es_host')
port = data.get('es_port')
username = data.get('es_username')
password = data.get('es_password')
url_prefix = data.get('es_url_prefix', '')
use_ssl = data.get('use_ssl')
else:
host = args.host if args.host else raw_input('Enter elasticsearch host: ')
port = args.port if args.port else int(raw_input('Enter elasticsearch port: '))
use_ssl = (args.ssl if args.ssl is not None
else args.no_ssl if args.no_ssl is not None
else raw_input('Use SSL? t/f: ').lower() in ('t', 'true'))
if args.no_auth is None:
username = raw_input('Enter optional basic-auth username: '******'Enter optional basic-auth password: '******'Enter optional Elasticsearch URL prefix: '))
if username and password:
http_auth = username + ':' + password
es = Elasticsearch(host=host, port=port, use_ssl=use_ssl, http_auth=http_auth, url_prefix=url_prefix)
silence_mapping = {'silence': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'until': {'type': 'date', 'format': 'dateOptionalTime'}}}}
ess_mapping = {'elastalert_status': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'@timestamp': {'format': 'dateOptionalTime', 'type': 'date'}}}}
es_mapping = {'elastalert': {'properties': {'rule_name': {'index': 'not_analyzed', 'type': 'string'},
'match_body': {'enabled': False, 'type': 'object'},
'aggregate_id': {'index': 'not_analyzed', 'type': 'string'}}}}
error_mapping = {'elastalert_error': {'properties': {'data': {'type': 'object', 'enabled': False}}}}
index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ')
if not index:
index = 'elastalert_status'
old_index = (args.old_index if args.old_index is not None
else raw_input('Name of existing index to copy? (Default None) '))
res = None
if old_index:
print('Downloading existing data...')
res = es.search(index=old_index, body={}, size=500000)
print('Got %s documents' % (len(res['hits']['hits'])))
es.indices.create(index)
es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_status', body=ess_mapping)
es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping)
es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping)
print('New index %s created' % (index))
if res:
bulk = ''.join(['%s\n%s\n' % (json.dumps({'create': {'_type': doc['_type'], '_index': index}}),
json.dumps(doc['_source'])) for doc in res['hits']['hits']])
print('Uploading data...')
es.bulk(body=bulk, index=index)
print('Done!')
def main(in_args=None):
parser = argparse.ArgumentParser()
parser.add_argument("--host", help="Elasticsearch host")
parser.add_argument("--port", type=int, help="Elasticsearch port")
parser.add_argument("--url-prefix", help="Elasticsearch URL prefix")
parser.add_argument("--no-auth", action="store_const", const=True, help="Suppress prompt for basic auth")
parser.add_argument("--ssl", action="store_true", default=None, help="Use SSL")
parser.add_argument("--no-ssl", dest="ssl", action="store_false", help="Do not use SSL")
parser.add_argument("--index", help="Index name to create")
parser.add_argument("--old-index", help="Old index name to copy")
parser.add_argument("--config", help="Config file name")
args = parser.parse_args(in_args)
if args.config:
filename = args.config
elif os.path.isfile("../config.yaml"):
filename = "../config.yaml"
elif os.path.isfile("config.yaml"):
filename = "config.yaml"
else:
filename = ""
username = None
password = None
use_ssl = None
url_prefix = None
http_auth = None
if filename:
with open(filename) as config_file:
data = yaml.load(config_file)
host = data.get("es_host")
port = data.get("es_port")
username = data.get("es_username")
password = data.get("es_password")
url_prefix = data.get("es_url_prefix", "")
use_ssl = data.get("use_ssl")
else:
host = args.host if args.host else raw_input("Enter elasticsearch host: ")
port = args.port if args.port else int(raw_input("Enter elasticsearch port: "))
use_ssl = args.ssl if args.ssl is not None else raw_input("Use SSL? t/f: ").lower() in ("t", "true")
if args.no_auth is None:
username = raw_input("Enter optional basic-auth username: "******"Enter optional basic-auth password: "******"Enter optional Elasticsearch URL prefix: ")
)
if username and password:
http_auth = username + ":" + password
es = Elasticsearch(host=host, port=port, use_ssl=use_ssl, http_auth=http_auth, url_prefix=url_prefix)
silence_mapping = {
"silence": {
"properties": {
"rule_name": {"index": "not_analyzed", "type": "string"},
"until": {"type": "date", "format": "dateOptionalTime"},
}
}
}
ess_mapping = {
"elastalert_status": {
"properties": {
"rule_name": {"index": "not_analyzed", "type": "string"},
"@timestamp": {"format": "dateOptionalTime", "type": "date"},
}
}
}
es_mapping = {
"elastalert": {
"properties": {
"rule_name": {"index": "not_analyzed", "type": "string"},
"match_body": {"enabled": False, "type": "object"},
"aggregate_id": {"index": "not_analyzed", "type": "string"},
}
}
}
error_mapping = {"elastalert_error": {"properties": {"data": {"type": "object", "enabled": False}}}}
index = args.index if args.index is not None else raw_input("New index name? (Default elastalert_status) ")
if not index:
index = "elastalert_status"
res = None
if args.old_index:
print("Downloading existing data...")
res = es.search(index=args.old_index, body={}, size=500000)
print("Got %s documents" % (len(res["hits"]["hits"])))
es.indices.create(index)
es.indices.put_mapping(index=index, doc_type="elastalert", body=es_mapping)
es.indices.put_mapping(index=index, doc_type="elastalert_status", body=ess_mapping)
es.indices.put_mapping(index=index, doc_type="silence", body=silence_mapping)
es.indices.put_mapping(index=index, doc_type="elastalert_error", body=error_mapping)
print("New index %s created" % (index))
if res:
bulk = "".join(
[
"%s\n%s\n"
% (json.dumps({"create": {"_type": doc["_type"], "_index": index}}), json.dumps(doc["_source"]))
for doc in res["hits"]["hits"]
]
)
print("Uploading data...")
es.bulk(body=bulk, index=index)
print("Done!")