def exec_request(esconfig: ESConfig, _oper, urlitems, params={}, body=None): client = Elasticsearch(esconfig.http_connect) if not isinstance(urlitems, (list, tuple)): urlitems = (urlitems, ) return client.transport.perform_request(_oper, _make_path(*urlitems), body=body)
def __call__(self, es, params): from enum import Enum from functools import total_ordering from elasticsearch.client import _make_path @total_ordering class ClusterHealthStatus(Enum): UNKNOWN = 0 RED = 1 YELLOW = 2 GREEN = 3 def __lt__(self, other): if self.__class__ is other.__class__: return self.value < other.value return NotImplemented def status(v): try: return ClusterHealthStatus[v.upper()] except (KeyError, AttributeError): return ClusterHealthStatus.UNKNOWN index = params.get("index") request_params = params.get("request-params", {}) # by default, Elasticsearch will not wait and thus we treat this as success expected_cluster_status = request_params.get( "wait_for_status", str(ClusterHealthStatus.UNKNOWN)) # newer ES versions >= 5.0 if "wait_for_no_relocating_shards" in request_params: expected_relocating_shards = 0 else: # older ES versions # either the user has defined something or we're good with any count of relocating shards. expected_relocating_shards = int( request_params.get("wait_for_relocating_shards", sys.maxsize)) # This would not work if the request parameter is not a proper method parameter for the ES client... # result = es.cluster.health(**request_params) result = es.transport.perform_request("GET", _make_path( "_cluster", "health", index), params=request_params) cluster_status = result["status"] relocating_shards = result["relocating_shards"] return { "weight": 1, "unit": "ops", "success": status(cluster_status) >= status(expected_cluster_status) and relocating_shards <= expected_relocating_shards, "cluster-status": cluster_status, "relocating-shards": relocating_shards }
def search(self, index=None, doc_type=None, body=None, params=None): # from is a reserved word so it cannot be used, use from_ instead if 'from_' in params: params['from'] = params.pop('from_') if doc_type and not index: index = '_all' return self.transport.perform_request('GET', _make_path( index, doc_type, '_search'), params=params, body=body, headers=self.headers)
def getClusterState(esconfig: ESConfig): log.debug('begin to check es cluster info') connects = [k for k in esconfig.http_connect] client = Elasticsearch(connects) try: cluster = client.cluster.health() nodes = client.transport.perform_request('GET', _make_path('_cat', 'nodes'), params={'v': 'y'}) nodes = nodes.splitlines() except Exception as e: log.error(format_exc()) else: log.debug(cluster.__repr__()) for node in nodes: log.debug(node)
def exec_restore(config: ESSnapshotRestoreConfig, repository, snapshotid): client = Elasticsearch(config.http_connect) body = {} checkAndSet(config, 'index_name', body, 'indices') checkAndSet(config, 'rename_pattern', body) checkAndSet(config, 'ignore_unavailable', body) checkAndSet(config, 'include_global_state', body) checkAndSet(config, 'partial', body) checkAndSet(config, 'index_settings', body) checkAndSet(config, 'include_aliases', body) checkAndSet(config, 'ignore_index_settings', body) checkAndSet(config, 'rename_replacement', body) body = expand(body) log.debug(config.http_connect) log.debug( '%s %s' % ('POST', _make_path('_snapshot', repository, snapshotid, '_restore'))) log.debug(body) res = client.snapshot.restore(repository, snapshotid, body) return res
def file_exec(client, file, config, notifier=None): if not os.path.exists(file): return filename = os.path.splitext(os.path.split(file)[1])[0] items = filename.split('#') _oper = items[1] items = items[2:] _url = _make_path(*items) _body = None if not _oper.upper() == 'DELETE': with open(file, 'r', encoding='utf-8') as f: _body = f.read() if config.log_statement: log.info(_url) log.info(_body) try: client.transport.perform_request(_oper, _url, body=_body) except ElasticsearchException as e: if isinstance(e, TransportError): log.error('status:%d, errorinfo: %s' % (e.status_code, e.error)) else: log.error(traceback.format_exc()) return -1 return 0
def deprecated_search(self, index=None, doc_type=None, body=None, params=None): """ Execute a search query and get back search hits that match the query. `<https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-search.html>`_ :arg index: A list of index names to search, or a string containing a comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices :arg doc_type: A comma-separated list of document types to search; leave empty to perform the operation on all types :arg body: The search definition using the Query DSL :arg _source: True or false to return the _source field or not, or a list of fields to return :arg _source_exclude: A list of fields to exclude from the returned _source field :arg _source_include: A list of fields to extract and return from the _source field :arg allow_no_indices: Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) :arg allow_partial_search_results: Set to false to return an overall failure if the request would produce partial results. Defaults to True, which will allow partial results in the case of timeouts or partial failures :arg analyze_wildcard: Specify whether wildcard and prefix queries should be analyzed (default: false) :arg analyzer: The analyzer to use for the query string :arg batched_reduce_size: The number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large., default 512 :arg default_operator: The default operator for query string query (AND or OR), default 'OR', valid choices are: 'AND', 'OR' :arg df: The field to use as default where no field prefix is given in the query string :arg docvalue_fields: A comma-separated list of fields to return as the docvalue representation of a field for each hit :arg expand_wildcards: Whether to expand wildcard expression to concrete indices that are open, closed or both., default 'open', valid choices are: 'open', 'closed', 'none', 'all' :arg explain: Specify whether to return detailed information about score computation as part of a hit :arg from\\_: Starting offset (default: 0) :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) :arg lenient: Specify whether format-based query failures (such as providing text to a numeric field) should be ignored :arg max_concurrent_shard_requests: The number of concurrent shard requests this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests, default 'The default grows with the number of nodes in the cluster but is at most 256.' :arg pre_filter_shard_size: A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint., default 128 :arg preference: Specify the node or shard the operation should be performed on (default: random) :arg q: Query in the Lucene query string syntax :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number in the response. This param is added version 6.x to handle mixed cluster queries where nodes are in multiple versions (7.0 and 6.latest) :arg request_cache: Specify if request cache should be used for this request or not, defaults to index level setting :arg routing: A comma-separated list of specific routing values :arg scroll: Specify how long a consistent view of the index should be maintained for scrolled search :arg search_type: Search operation type, valid choices are: 'query_then_fetch', 'dfs_query_then_fetch' :arg size: Number of hits to return (default: 10) :arg sort: A comma-separated list of <field>:<direction> pairs :arg stats: Specific 'tag' of the request for logging and statistical purposes :arg stored_fields: A comma-separated list of stored fields to return as part of a hit :arg suggest_field: Specify which field to use for suggestions :arg suggest_mode: Specify suggest mode, default 'missing', valid choices are: 'missing', 'popular', 'always' :arg suggest_size: How many suggestions to return in response :arg suggest_text: The source text for which the suggestions should be returned :arg terminate_after: The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early. :arg timeout: Explicit operation timeout :arg track_scores: Whether to calculate and return scores even if they are not used for sorting :arg track_total_hits: Indicate if the number of documents that match the query should be tracked :arg typed_keys: Specify whether aggregation and suggester names should be prefixed by their respective types in the response :arg version: Specify whether to return document version as part of a hit """ # from is a reserved word so it cannot be used, use from_ instead if "from_" in params: params["from"] = params.pop("from_") if not index: index = "_all" res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body ) if type(res) == list or type(res) == tuple: return res[1] return res
parser.add_argument("--es_password", default="changeme", help="ES Password") parser.add_argument("--interval", default=300, help="Interval in Seconds", type=int) parser.add_argument("--start_time", help="Start Time") parser.add_argument("--end_time", help="End Time") parser.add_argument("--watch_template", help="Watch File") options = parser.parse_args() start_time = datetime.strptime(options.start_time, '%Y-%m-%dT%H:%M:%SZ') end_time = datetime.strptime(options.end_time, '%Y-%m-%dT%H:%M:%SZ') client = Elasticsearch(hosts=[options.es_host], http_auth=(options.es_user, options.es_password), use_ssl=False, timeout=300) try: cluster = client.info() except: print("Cluster not accessible") sys.exit(1) watch_template = json.loads(open(options.watch_template).read()) next_time = start_time while next_time < end_time: print("Executing for %s-%s seconds"%(next_time.strftime('%Y-%m-%dT%H:%M:%SZ'),options.interval)) watch_body = watch_template watch_body["metadata"]["time_period"] = "%ss"%options.interval client.transport.perform_request('POST', _make_path('_xpack', 'watcher', 'watch', '_execute'), body={ "trigger_data":{ "scheduled_time":next_time.strftime('%Y-%m-%dT%H:%M:%SZ') }, "watch":watch_body }) next_time = next_time + timedelta(seconds=options.interval)
def deprecated_search(self, index=None, doc_type=None, body=None, params=None): """ Execute a search query and get back search hits that match the query. `<https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-search.html>`_ :arg index: A list of index names to search, or a string containing a comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices :arg doc_type: A comma-separated list of document types to search; leave empty to perform the operation on all types :arg body: The search definition using the Query DSL :arg _source: True or false to return the _source field or not, or a list of fields to return :arg _source_exclude: A list of fields to exclude from the returned _source field :arg _source_include: A list of fields to extract and return from the _source field :arg allow_no_indices: Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) :arg allow_partial_search_results: Set to false to return an overall failure if the request would produce partial results. Defaults to True, which will allow partial results in the case of timeouts or partial failures :arg analyze_wildcard: Specify whether wildcard and prefix queries should be analyzed (default: false) :arg analyzer: The analyzer to use for the query string :arg batched_reduce_size: The number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large., default 512 :arg default_operator: The default operator for query string query (AND or OR), default 'OR', valid choices are: 'AND', 'OR' :arg df: The field to use as default where no field prefix is given in the query string :arg docvalue_fields: A comma-separated list of fields to return as the docvalue representation of a field for each hit :arg expand_wildcards: Whether to expand wildcard expression to concrete indices that are open, closed or both., default 'open', valid choices are: 'open', 'closed', 'none', 'all' :arg explain: Specify whether to return detailed information about score computation as part of a hit :arg from\\_: Starting offset (default: 0) :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) :arg lenient: Specify whether format-based query failures (such as providing text to a numeric field) should be ignored :arg max_concurrent_shard_requests: The number of concurrent shard requests this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests, default 'The default grows with the number of nodes in the cluster but is at most 256.' :arg pre_filter_shard_size: A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint., default 128 :arg preference: Specify the node or shard the operation should be performed on (default: random) :arg q: Query in the Lucene query string syntax :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number in the response. This param is added version 6.x to handle mixed cluster queries where nodes are in multiple versions (7.0 and 6.latest) :arg request_cache: Specify if request cache should be used for this request or not, defaults to index level setting :arg routing: A comma-separated list of specific routing values :arg scroll: Specify how long a consistent view of the index should be maintained for scrolled search :arg search_type: Search operation type, valid choices are: 'query_then_fetch', 'dfs_query_then_fetch' :arg size: Number of hits to return (default: 10) :arg sort: A comma-separated list of <field>:<direction> pairs :arg stats: Specific 'tag' of the request for logging and statistical purposes :arg stored_fields: A comma-separated list of stored fields to return as part of a hit :arg suggest_field: Specify which field to use for suggestions :arg suggest_mode: Specify suggest mode, default 'missing', valid choices are: 'missing', 'popular', 'always' :arg suggest_size: How many suggestions to return in response :arg suggest_text: The source text for which the suggestions should be returned :arg terminate_after: The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early. :arg timeout: Explicit operation timeout :arg track_scores: Whether to calculate and return scores even if they are not used for sorting :arg track_total_hits: Indicate if the number of documents that match the query should be tracked :arg typed_keys: Specify whether aggregation and suggester names should be prefixed by their respective types in the response :arg version: Specify whether to return document version as part of a hit """ logging.warning( 'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8') # from is a reserved word so it cannot be used, use from_ instead if "from_" in params: params["from"] = params.pop("from_") if not index: index = "_all" return self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body )
options = parser.parse_args() start_time = datetime.strptime(options.start_time, '%Y-%m-%dT%H:%M:%SZ') end_time = datetime.strptime(options.end_time, '%Y-%m-%dT%H:%M:%SZ') client = Elasticsearch(hosts=[options.es_host], http_auth=(options.es_user, options.es_password), use_ssl=False, timeout=300) try: cluster = client.info() except: print("Cluster not accessible") sys.exit(1) watch_template = json.loads(open(options.watch_template).read()) next_time = start_time while next_time < end_time: print("Executing for %s-%s seconds" % (next_time.strftime('%Y-%m-%dT%H:%M:%SZ'), options.interval)) watch_body = watch_template watch_body["metadata"]["time_period"] = "%ss" % options.interval client.transport.perform_request( 'POST', _make_path('_xpack', 'watcher', 'watch', '_execute'), body={ "trigger_data": { "scheduled_time": next_time.strftime('%Y-%m-%dT%H:%M:%SZ') }, "watch": watch_body }) next_time = next_time + timedelta(seconds=options.interval)