def exec_request(esconfig: ESConfig, _oper, urlitems, params={}, body=None):
client = Elasticsearch(esconfig.http_connect)
if not isinstance(urlitems, (list, tuple)):
urlitems = (urlitems, )
return client.transport.perform_request(_oper,
_make_path(*urlitems),
body=body)
def __call__(self, es, params):
from enum import Enum
from functools import total_ordering
from elasticsearch.client import _make_path
@total_ordering
class ClusterHealthStatus(Enum):
UNKNOWN = 0
RED = 1
YELLOW = 2
GREEN = 3
def __lt__(self, other):
if self.__class__ is other.__class__:
return self.value < other.value
return NotImplemented
def status(v):
try:
return ClusterHealthStatus[v.upper()]
except (KeyError, AttributeError):
return ClusterHealthStatus.UNKNOWN
index = params.get("index")
request_params = params.get("request-params", {})
# by default, Elasticsearch will not wait and thus we treat this as success
expected_cluster_status = request_params.get(
"wait_for_status", str(ClusterHealthStatus.UNKNOWN))
# newer ES versions >= 5.0
if "wait_for_no_relocating_shards" in request_params:
expected_relocating_shards = 0
else:
# older ES versions
# either the user has defined something or we're good with any count of relocating shards.
expected_relocating_shards = int(
request_params.get("wait_for_relocating_shards", sys.maxsize))
# This would not work if the request parameter is not a proper method parameter for the ES client...
# result = es.cluster.health(**request_params)
result = es.transport.perform_request("GET",
_make_path(
"_cluster", "health", index),
params=request_params)
cluster_status = result["status"]
relocating_shards = result["relocating_shards"]
return {
"weight":
1,
"unit":
"ops",
"success":
status(cluster_status) >= status(expected_cluster_status)
and relocating_shards <= expected_relocating_shards,
"cluster-status":
cluster_status,
"relocating-shards":
relocating_shards
}
def search(self, index=None, doc_type=None, body=None, params=None):
# from is a reserved word so it cannot be used, use from_ instead
if 'from_' in params:
params['from'] = params.pop('from_')
if doc_type and not index:
index = '_all'
return self.transport.perform_request('GET',
_make_path(
index, doc_type,
'_search'),
params=params,
body=body,
headers=self.headers)
def getClusterState(esconfig: ESConfig):
log.debug('begin to check es cluster info')
connects = [k for k in esconfig.http_connect]
client = Elasticsearch(connects)
try:
cluster = client.cluster.health()
nodes = client.transport.perform_request('GET',
_make_path('_cat', 'nodes'),
params={'v': 'y'})
nodes = nodes.splitlines()
except Exception as e:
log.error(format_exc())
else:
log.debug(cluster.__repr__())
for node in nodes:
log.debug(node)
def exec_restore(config: ESSnapshotRestoreConfig, repository, snapshotid):
client = Elasticsearch(config.http_connect)
body = {}
checkAndSet(config, 'index_name', body, 'indices')
checkAndSet(config, 'rename_pattern', body)
checkAndSet(config, 'ignore_unavailable', body)
checkAndSet(config, 'include_global_state', body)
checkAndSet(config, 'partial', body)
checkAndSet(config, 'index_settings', body)
checkAndSet(config, 'include_aliases', body)
checkAndSet(config, 'ignore_index_settings', body)
checkAndSet(config, 'rename_replacement', body)
body = expand(body)
log.debug(config.http_connect)
log.debug(
'%s %s' %
('POST', _make_path('_snapshot', repository, snapshotid, '_restore')))
log.debug(body)
res = client.snapshot.restore(repository, snapshotid, body)
return res
def file_exec(client, file, config, notifier=None):
if not os.path.exists(file):
return
filename = os.path.splitext(os.path.split(file)[1])[0]
items = filename.split('#')
_oper = items[1]
items = items[2:]
_url = _make_path(*items)
_body = None
if not _oper.upper() == 'DELETE':
with open(file, 'r', encoding='utf-8') as f:
_body = f.read()
if config.log_statement:
log.info(_url)
log.info(_body)
try:
client.transport.perform_request(_oper, _url, body=_body)
except ElasticsearchException as e:
if isinstance(e, TransportError):
log.error('status:%d, errorinfo: %s' % (e.status_code, e.error))
else:
log.error(traceback.format_exc())
return -1
return 0
def deprecated_search(self, index=None, doc_type=None, body=None, params=None):
"""
Execute a search query and get back search hits that match the query.
`<https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-search.html>`_
:arg index: A list of index names to search, or a string containing a
comma-separated list of index names to search; use `_all`
or empty string to perform the operation on all indices
:arg doc_type: A comma-separated list of document types to search; leave
empty to perform the operation on all types
:arg body: The search definition using the Query DSL
:arg _source: True or false to return the _source field or not, or a
list of fields to return
:arg _source_exclude: A list of fields to exclude from the returned
_source field
:arg _source_include: A list of fields to extract and return from the
_source field
:arg allow_no_indices: Whether to ignore if a wildcard indices
expression resolves into no concrete indices. (This includes `_all`
string or when no indices have been specified)
:arg allow_partial_search_results: Set to false to return an overall
failure if the request would produce partial results. Defaults to
True, which will allow partial results in the case of timeouts or
partial failures
:arg analyze_wildcard: Specify whether wildcard and prefix queries
should be analyzed (default: false)
:arg analyzer: The analyzer to use for the query string
:arg batched_reduce_size: The number of shard results that should be
reduced at once on the coordinating node. This value should be used
as a protection mechanism to reduce the memory overhead per search
request if the potential number of shards in the request can be
large., default 512
:arg default_operator: The default operator for query string query (AND
or OR), default 'OR', valid choices are: 'AND', 'OR'
:arg df: The field to use as default where no field prefix is given in
the query string
:arg docvalue_fields: A comma-separated list of fields to return as the
docvalue representation of a field for each hit
:arg expand_wildcards: Whether to expand wildcard expression to concrete
indices that are open, closed or both., default 'open', valid
choices are: 'open', 'closed', 'none', 'all'
:arg explain: Specify whether to return detailed information about score
computation as part of a hit
:arg from\\_: Starting offset (default: 0)
:arg ignore_unavailable: Whether specified concrete indices should be
ignored when unavailable (missing or closed)
:arg lenient: Specify whether format-based query failures (such as
providing text to a numeric field) should be ignored
:arg max_concurrent_shard_requests: The number of concurrent shard
requests this search executes concurrently. This value should be
used to limit the impact of the search on the cluster in order to
limit the number of concurrent shard requests, default 'The default
grows with the number of nodes in the cluster but is at most 256.'
:arg pre_filter_shard_size: A threshold that enforces a pre-filter
roundtrip to prefilter search shards based on query rewriting if
the number of shards the search request expands to exceeds the
threshold. This filter roundtrip can limit the number of shards
significantly if for instance a shard can not match any documents
based on it's rewrite method ie. if date filters are mandatory to
match but the shard bounds and the query are disjoint., default 128
:arg preference: Specify the node or shard the operation should be
performed on (default: random)
:arg q: Query in the Lucene query string syntax
:arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number
in the response. This param is added version 6.x to handle mixed cluster queries where nodes
are in multiple versions (7.0 and 6.latest)
:arg request_cache: Specify if request cache should be used for this
request or not, defaults to index level setting
:arg routing: A comma-separated list of specific routing values
:arg scroll: Specify how long a consistent view of the index should be
maintained for scrolled search
:arg search_type: Search operation type, valid choices are:
'query_then_fetch', 'dfs_query_then_fetch'
:arg size: Number of hits to return (default: 10)
:arg sort: A comma-separated list of <field>:<direction> pairs
:arg stats: Specific 'tag' of the request for logging and statistical
purposes
:arg stored_fields: A comma-separated list of stored fields to return as
part of a hit
:arg suggest_field: Specify which field to use for suggestions
:arg suggest_mode: Specify suggest mode, default 'missing', valid
choices are: 'missing', 'popular', 'always'
:arg suggest_size: How many suggestions to return in response
:arg suggest_text: The source text for which the suggestions should be
returned
:arg terminate_after: The maximum number of documents to collect for
each shard, upon reaching which the query execution will terminate
early.
:arg timeout: Explicit operation timeout
:arg track_scores: Whether to calculate and return scores even if they
are not used for sorting
:arg track_total_hits: Indicate if the number of documents that match
the query should be tracked
:arg typed_keys: Specify whether aggregation and suggester names should
be prefixed by their respective types in the response
:arg version: Specify whether to return document version as part of a
hit
"""
# from is a reserved word so it cannot be used, use from_ instead
if "from_" in params:
params["from"] = params.pop("from_")
if not index:
index = "_all"
res = self.transport.perform_request(
"GET", _make_path(index, doc_type, "_search"), params=params, body=body
)
if type(res) == list or type(res) == tuple:
return res[1]
return res
parser.add_argument("--es_password", default="changeme", help="ES Password")
parser.add_argument("--interval", default=300, help="Interval in Seconds", type=int)
parser.add_argument("--start_time", help="Start Time")
parser.add_argument("--end_time", help="End Time")
parser.add_argument("--watch_template", help="Watch File")
options = parser.parse_args()
start_time = datetime.strptime(options.start_time, '%Y-%m-%dT%H:%M:%SZ')
end_time = datetime.strptime(options.end_time, '%Y-%m-%dT%H:%M:%SZ')
client = Elasticsearch(hosts=[options.es_host], http_auth=(options.es_user, options.es_password), use_ssl=False,
timeout=300)
try:
cluster = client.info()
except:
print("Cluster not accessible")
sys.exit(1)
watch_template = json.loads(open(options.watch_template).read())
next_time = start_time
while next_time < end_time:
print("Executing for %s-%s seconds"%(next_time.strftime('%Y-%m-%dT%H:%M:%SZ'),options.interval))
watch_body = watch_template
watch_body["metadata"]["time_period"] = "%ss"%options.interval
client.transport.perform_request('POST', _make_path('_xpack',
'watcher', 'watch', '_execute'), body={
"trigger_data":{
"scheduled_time":next_time.strftime('%Y-%m-%dT%H:%M:%SZ')
},
"watch":watch_body
})
next_time = next_time + timedelta(seconds=options.interval)
def deprecated_search(self, index=None, doc_type=None, body=None, params=None):
"""
Execute a search query and get back search hits that match the query.
`<https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-search.html>`_
:arg index: A list of index names to search, or a string containing a
comma-separated list of index names to search; use `_all`
or empty string to perform the operation on all indices
:arg doc_type: A comma-separated list of document types to search; leave
empty to perform the operation on all types
:arg body: The search definition using the Query DSL
:arg _source: True or false to return the _source field or not, or a
list of fields to return
:arg _source_exclude: A list of fields to exclude from the returned
_source field
:arg _source_include: A list of fields to extract and return from the
_source field
:arg allow_no_indices: Whether to ignore if a wildcard indices
expression resolves into no concrete indices. (This includes `_all`
string or when no indices have been specified)
:arg allow_partial_search_results: Set to false to return an overall
failure if the request would produce partial results. Defaults to
True, which will allow partial results in the case of timeouts or
partial failures
:arg analyze_wildcard: Specify whether wildcard and prefix queries
should be analyzed (default: false)
:arg analyzer: The analyzer to use for the query string
:arg batched_reduce_size: The number of shard results that should be
reduced at once on the coordinating node. This value should be used
as a protection mechanism to reduce the memory overhead per search
request if the potential number of shards in the request can be
large., default 512
:arg default_operator: The default operator for query string query (AND
or OR), default 'OR', valid choices are: 'AND', 'OR'
:arg df: The field to use as default where no field prefix is given in
the query string
:arg docvalue_fields: A comma-separated list of fields to return as the
docvalue representation of a field for each hit
:arg expand_wildcards: Whether to expand wildcard expression to concrete
indices that are open, closed or both., default 'open', valid
choices are: 'open', 'closed', 'none', 'all'
:arg explain: Specify whether to return detailed information about score
computation as part of a hit
:arg from\\_: Starting offset (default: 0)
:arg ignore_unavailable: Whether specified concrete indices should be
ignored when unavailable (missing or closed)
:arg lenient: Specify whether format-based query failures (such as
providing text to a numeric field) should be ignored
:arg max_concurrent_shard_requests: The number of concurrent shard
requests this search executes concurrently. This value should be
used to limit the impact of the search on the cluster in order to
limit the number of concurrent shard requests, default 'The default
grows with the number of nodes in the cluster but is at most 256.'
:arg pre_filter_shard_size: A threshold that enforces a pre-filter
roundtrip to prefilter search shards based on query rewriting if
the number of shards the search request expands to exceeds the
threshold. This filter roundtrip can limit the number of shards
significantly if for instance a shard can not match any documents
based on it's rewrite method ie. if date filters are mandatory to
match but the shard bounds and the query are disjoint., default 128
:arg preference: Specify the node or shard the operation should be
performed on (default: random)
:arg q: Query in the Lucene query string syntax
:arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number
in the response. This param is added version 6.x to handle mixed cluster queries where nodes
are in multiple versions (7.0 and 6.latest)
:arg request_cache: Specify if request cache should be used for this
request or not, defaults to index level setting
:arg routing: A comma-separated list of specific routing values
:arg scroll: Specify how long a consistent view of the index should be
maintained for scrolled search
:arg search_type: Search operation type, valid choices are:
'query_then_fetch', 'dfs_query_then_fetch'
:arg size: Number of hits to return (default: 10)
:arg sort: A comma-separated list of <field>:<direction> pairs
:arg stats: Specific 'tag' of the request for logging and statistical
purposes
:arg stored_fields: A comma-separated list of stored fields to return as
part of a hit
:arg suggest_field: Specify which field to use for suggestions
:arg suggest_mode: Specify suggest mode, default 'missing', valid
choices are: 'missing', 'popular', 'always'
:arg suggest_size: How many suggestions to return in response
:arg suggest_text: The source text for which the suggestions should be
returned
:arg terminate_after: The maximum number of documents to collect for
each shard, upon reaching which the query execution will terminate
early.
:arg timeout: Explicit operation timeout
:arg track_scores: Whether to calculate and return scores even if they
are not used for sorting
:arg track_total_hits: Indicate if the number of documents that match
the query should be tracked
:arg typed_keys: Specify whether aggregation and suggester names should
be prefixed by their respective types in the response
:arg version: Specify whether to return document version as part of a
hit
"""
logging.warning(
'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8')
# from is a reserved word so it cannot be used, use from_ instead
if "from_" in params:
params["from"] = params.pop("from_")
if not index:
index = "_all"
return self.transport.perform_request(
"GET", _make_path(index, doc_type, "_search"), params=params, body=body
)
options = parser.parse_args()
start_time = datetime.strptime(options.start_time, '%Y-%m-%dT%H:%M:%SZ')
end_time = datetime.strptime(options.end_time, '%Y-%m-%dT%H:%M:%SZ')
client = Elasticsearch(hosts=[options.es_host],
http_auth=(options.es_user, options.es_password),
use_ssl=False,
timeout=300)
try:
cluster = client.info()
except:
print("Cluster not accessible")
sys.exit(1)
watch_template = json.loads(open(options.watch_template).read())
next_time = start_time
while next_time < end_time:
print("Executing for %s-%s seconds" %
(next_time.strftime('%Y-%m-%dT%H:%M:%SZ'), options.interval))
watch_body = watch_template
watch_body["metadata"]["time_period"] = "%ss" % options.interval
client.transport.perform_request(
'POST',
_make_path('_xpack', 'watcher', 'watch', '_execute'),
body={
"trigger_data": {
"scheduled_time": next_time.strftime('%Y-%m-%dT%H:%M:%SZ')
},
"watch": watch_body
})
next_time = next_time + timedelta(seconds=options.interval)
