class ElasticsearchSampler(): """Elasticsearchサンプルクラス """ def __init__(self): host = 'localhost' port = 9200 auth = ('admin', 'admin') # certs = 'esnode.pem' # Elasticsearchインタンスの作成 self.es = OpenSearch( hosts=[{'host': host, 'port': port}], http_auth=auth, use_ssl=True, verify_certs=False, # ca_certs=certs, ssl_assert_hostname=False, ssl_show_warn=False, ) def __del__(self): self.es.close() print("close elasticsearch instance--------------------------") def search(self, idx: str, query: str): """検索 """ result = self.es.search(index=idx, body=query) print('--[search]-------------------------------------------') pprint.pprint(result, sort_dicts=False) def bulk(self, index: str): """バルクインサート """ try: # iterableなオブジェクトであればよいので以下どちらも可能 # - ジェネレータで渡す success, failed = helpers.bulk(self.es, gendata3(index)) # - list型で渡す # success, failed = helpers.bulk(self.es, bulklist()) # except opensearchpy.ElasticsearchException as e: # pprint.pprint(e) except Exception as e: pprint.pprint(e) return print('--[bulk ]-------------------------------------------') pprint.pprint(success) pprint.pprint(failed) def delete_by_query(self, idx: str, query: str): """条件指定の削除 """ result = self.es.delete_by_query(index=idx, body=query) print(f'{type(result)}') print('--[delete_by_query]----------------------------------') pprint.pprint(result, sort_dicts=False)
class ElasticSearchDB(object): """ .. class:: ElasticSearchDB :param str url: the url to the database for example: el.cern.ch:9200 :param str gDebugFile: is used to save the debug information to a file :param int timeout: the default time out to Elasticsearch :param int RESULT_SIZE: The number of data points which will be returned by the query. """ __url = "" __timeout = 120 clusterName = "" RESULT_SIZE = 10000 ######################################################################## def __init__( self, host, port, user=None, password=None, indexPrefix="", useSSL=True, useCRT=False, ca_certs=None, client_key=None, client_cert=None, ): """c'tor :param self: self reference :param str host: name of the database for example: MonitoringDB :param str port: The full name of the database for example: 'Monitoring/MonitoringDB' :param str user: user name to access the db :param str password: if the db is password protected we need to provide a password :param str indexPrefix: it is the indexPrefix used to get all indexes :param bool useSSL: We can disable using secure connection. By default we use secure connection. :param bool useCRT: Use certificates. :param str ca_certs: CA certificates bundle. :param str client_key: Client key. :param str client_cert: Client certificate. """ self.__indexPrefix = indexPrefix self._connected = False if user and password: sLog.debug("Specified username and password") if port: self.__url = "https://%s:%s@%s:%d" % (user, password, host, port) else: self.__url = "https://%s:%s@%s" % (user, password, host) else: sLog.debug("Username and password not specified") if port: self.__url = "http://%s:%d" % (host, port) else: self.__url = "http://%s" % host if port: sLog.verbose("Connecting to %s:%s, useSSL = %s" % (host, port, useSSL)) else: sLog.verbose("Connecting to %s, useSSL = %s" % (host, useSSL)) if useSSL: if ca_certs: casFile = ca_certs else: bd = BundleDeliveryClient() retVal = bd.getCAs() casFile = None if not retVal["OK"]: sLog.error("CAs file does not exists:", retVal["Message"]) casFile = certifi.where() else: casFile = retVal["Value"] self.client = Elasticsearch(self.__url, timeout=self.__timeout, use_ssl=True, verify_certs=True, ca_certs=casFile) elif useCRT: self.client = Elasticsearch( self.__url, timeout=self.__timeout, use_ssl=True, verify_certs=True, ca_certs=ca_certs, client_cert=client_cert, client_key=client_key, ) else: self.client = Elasticsearch(self.__url, timeout=self.__timeout) # Before we use the database we try to connect # and retrieve the cluster name try: if self.client.ping(): # Returns True if the cluster is running, False otherwise result = self.client.info() self.clusterName = result.get("cluster_name", " ") # pylint: disable=no-member sLog.info("Database info\n", json.dumps(result, indent=4)) self._connected = True else: sLog.error("Cannot ping ElasticsearchDB!") except ConnectionError as e: sLog.error(repr(e)) ######################################################################## def getIndexPrefix(self): """ It returns the DIRAC setup. """ return self.__indexPrefix ######################################################################## @ifConnected def query(self, index, query): """Executes a query and returns its result (uses ES DSL language). :param self: self reference :param str index: index name :param dict query: It is the query in ElasticSearch DSL language """ try: esDSLQueryResult = self.client.search(index=index, body=query) return S_OK(esDSLQueryResult) except RequestError as re: return S_ERROR(re) @ifConnected def update(self, index, query=None, updateByQuery=True, id=None): """Executes an update of a document, and returns S_OK/S_ERROR :param self: self reference :param str index: index name :param dict query: It is the query in ElasticSearch DSL language :param bool updateByQuery: A bool to determine update by query or index values using index function. :param int id: ID for the document to be created. """ sLog.debug("Updating %s with %s, updateByQuery=%s, id=%s" % (index, query, updateByQuery, id)) if not index or not query: return S_ERROR("Missing index or query") try: if updateByQuery: esDSLQueryResult = self.client.update_by_query(index=index, body=query) else: esDSLQueryResult = self.client.index(index=index, body=query, id=id) return S_OK(esDSLQueryResult) except RequestError as re: return S_ERROR(re) @ifConnected def _Search(self, indexname): """ it returns the object which can be used for retreiving certain value from the DB """ return Search(using=self.client, index=indexname) ######################################################################## def _Q(self, name_or_query="match", **params): """ It is a wrapper to ElasticDSL Query module used to create a query object. :param str name_or_query is the type of the query """ return Q(name_or_query, **params) def _A(self, name_or_agg, aggsfilter=None, **params): """ It is a wrapper to ElasticDSL aggregation module, used to create an aggregation """ return A(name_or_agg, aggsfilter, **params) ######################################################################## @ifConnected def getIndexes(self, indexName=None): """ It returns the available indexes... """ if not indexName: indexName = self.__indexPrefix sLog.debug("Getting indices alias of %s" % indexName) # we only return indexes which belong to a specific prefix for example 'lhcb-production' or 'dirac-production etc. return list(self.client.indices.get_alias("%s*" % indexName)) ######################################################################## @ifConnected def getDocTypes(self, indexName): """ Returns mappings, by index. :param str indexName: is the name of the index... :return: S_OK or S_ERROR """ result = [] try: sLog.debug("Getting mappings for ", indexName) result = self.client.indices.get_mapping(indexName) except Exception as e: # pylint: disable=broad-except sLog.exception() return S_ERROR(e) doctype = "" for indexConfig in result: if not result[indexConfig].get("mappings"): # there is a case when the mapping exits and the value is None... # this is usually an empty index or a corrupted index. sLog.warn("Index does not have mapping %s!" % indexConfig) continue if result[indexConfig].get("mappings"): doctype = result[indexConfig]["mappings"] break # we suppose the mapping of all indexes are the same... if not doctype: return S_ERROR("%s does not exists!" % indexName) return S_OK(doctype) ######################################################################## @ifConnected def existingIndex(self, indexName): """ Checks the existance of an index, by its name :param str indexName: the name of the index :returns: S_OK/S_ERROR if the request is successful """ sLog.debug("Checking existance of index %s" % indexName) try: return S_OK(self.client.indices.exists(indexName)) except TransportError as e: sLog.exception() return S_ERROR(e) ######################################################################## @ifConnected def createIndex(self, indexPrefix, mapping=None, period="day"): """ :param str indexPrefix: it is the index name. :param dict mapping: the configuration of the index. :param str period: We can specify, which kind of index will be created. Currently only daily and monthly indexes are supported. """ if period is not None: fullIndex = self.generateFullIndexName( indexPrefix, period) # we have to create an index each period... else: sLog.warn( "The period is not provided, so using non-periodic indexes names" ) fullIndex = indexPrefix res = self.existingIndex(fullIndex) if not res["OK"]: return res elif res["Value"]: return S_OK(fullIndex) try: sLog.info("Create index: ", fullIndex + str(mapping)) self.client.indices.create(index=fullIndex, body={"mappings": mapping}) # ES7 return S_OK(fullIndex) except Exception as e: # pylint: disable=broad-except sLog.error("Can not create the index:", repr(e)) return S_ERROR("Can not create the index") @ifConnected def deleteIndex(self, indexName): """ :param str indexName: the name of the index to be deleted... """ sLog.info("Deleting index", indexName) try: retVal = self.client.indices.delete(indexName) except NotFoundError: sLog.warn("Index does not exist", indexName) return S_OK("Noting to delete") except ValueError as e: return S_ERROR(DErrno.EVALUE, e) if retVal.get("acknowledged"): # if the value exists and the value is not None sLog.info("Deleted index", indexName) return S_OK(indexName) return S_ERROR(retVal) def index(self, indexName, body=None, docID=None, op_type="index"): """ :param str indexName: the name of the index to be used :param dict body: the data which will be indexed (basically the JSON) :param int id: optional document id :param str op_type: Explicit operation type. (options: 'index' (default) or 'create') :return: the index name in case of success. """ sLog.debug("Indexing in %s body %s, id=%s" % (indexName, body, docID)) if not indexName or not body: return S_ERROR("Missing index or body") try: res = self.client.index(index=indexName, body=body, id=docID, params={"op_type": op_type}) except (RequestError, TransportError) as e: sLog.exception() return S_ERROR(e) if res.get("created") or res.get("result") in ("created", "updated"): # the created index exists but the value can be None. return S_OK(indexName) return S_ERROR(res) @ifConnected def bulk_index(self, indexPrefix, data=None, mapping=None, period="day", withTimeStamp=True): """ :param str indexPrefix: index name. :param list data: contains a list of dictionary :param dict mapping: the mapping used by elasticsearch :param str period: Accepts 'day' and 'month'. We can specify which kind of indexes will be created. :param bool withTimeStamp: add timestamp to data, if not there already. :returns: S_OK/S_ERROR """ sLog.verbose("Bulk indexing", "%d records will be inserted" % len(data)) if mapping is None: mapping = {} if period is not None: indexName = self.generateFullIndexName(indexPrefix, period) else: indexName = indexPrefix sLog.debug("Bulk indexing into %s of %s" % (indexName, data)) res = self.existingIndex(indexName) if not res["OK"]: return res if not res["Value"]: retVal = self.createIndex(indexPrefix, mapping, period) if not retVal["OK"]: return retVal try: res = bulk(client=self.client, index=indexName, actions=generateDocs(data, withTimeStamp)) except (BulkIndexError, RequestError) as e: sLog.exception() return S_ERROR(e) if res[0] == len(data): # we have inserted all documents... return S_OK(len(data)) else: return S_ERROR(res) @ifConnected def getUniqueValue(self, indexName, key, orderBy=False): """ :param str indexName: the name of the index which will be used for the query :param dict orderBy: it is a dictionary in case we want to order the result {key:'desc'} or {key:'asc'} :returns: a list of unique value for a certain key from the dictionary. """ query = self._Search(indexName) endDate = datetime.utcnow() startDate = endDate - timedelta(days=30) timeFilter = self._Q( "range", timestamp={ "lte": int(TimeUtilities.toEpoch(endDate)) * 1000, "gte": int(TimeUtilities.toEpoch(startDate)) * 1000, }, ) query = query.filter("bool", must=timeFilter) if orderBy: query.aggs.bucket(key, "terms", field=key, size=self.RESULT_SIZE, order=orderBy).metric(key, "cardinality", field=key) else: query.aggs.bucket(key, "terms", field=key, size=self.RESULT_SIZE).metric(key, "cardinality", field=key) try: query = query.extra( size=self.RESULT_SIZE) # do not need the raw data. sLog.debug("Query", query.to_dict()) result = query.execute() except TransportError as e: return S_ERROR(e) values = [] for bucket in result.aggregations[key].buckets: values += [bucket["key"]] del query sLog.debug("Nb of unique rows retrieved", len(values)) return S_OK(values) def pingDB(self): """ Try to connect to the database :return: S_OK(TRUE/FALSE) """ connected = False try: connected = self.client.ping() except ConnectionError as e: sLog.error("Cannot connect to the db", repr(e)) return S_OK(connected) @ifConnected def deleteByQuery(self, indexName, query): """ Delete data by query (careful!) :param str indexName: the name of the index :param str query: the JSON-formatted query for which we want to issue the delete """ try: self.client.delete_by_query(index=indexName, body=query) except Exception as inst: sLog.error("ERROR: Couldn't delete data") return S_ERROR(inst) return S_OK("Successfully deleted data from index %s" % indexName) @staticmethod def generateFullIndexName(indexName, period): """ Given an index prefix we create the actual index name. :param str indexName: it is the name of the index :param str period: We can specify which kind of indexes will be created (day, week, month, year, null). :returns: string with full index name """ # if the period is not correct, we use no-period indexes (same as "null"). if period.lower() not in ["day", "week", "month", "year", "null"]: sLog.error("Period is not correct: ", period) return indexName elif period.lower() == "day": today = datetime.today().strftime("%Y-%m-%d") return "%s-%s" % (indexName, today) elif period.lower() == "week": week = datetime.today().isocalendar()[1] return "%s-%s" % (indexName, week) elif period.lower() == "month": month = datetime.today().strftime("%Y-%m") return "%s-%s" % (indexName, month) elif period.lower() == "year": year = datetime.today().strftime("%Y") return "%s-%s" % (indexName, year) elif period.lower() == "null": return indexName