def getESConn(): """ Get connection to Amazon Elasticsearch service (the casebase). Can be modified to point to any other Elasticsearch cluster. """ if(is_dev): return OpenSearch( hosts = [{'host': 'clood-opensearch', 'port': 9200}], http_compress = True, # enables gzip compression for request bodies http_auth = ('kibanaserver','kibanaserver'), use_ssl = False, verify_certs = False, ssl_assert_hostname = False, ssl_show_warn = False, ) esconn = OpenSearch( hosts=[{'host': host, 'port': 443}], http_auth=AWS4Auth(access_key, secret_key, region, 'es'), use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection ) return esconn
def __init__(self, host='127.0.0.1', port=9200, url=None): """Create an OpenSearch client.""" super().__init__() if url: self.client = OpenSearch([url], timeout=30) else: self.client = OpenSearch([{ 'host': host, 'port': port }], timeout=30) self.import_counter = collections.Counter() self.import_events = []
def build_es_connection(self): ''' Creates an Elasticsearch connection object that can be used to query Elasticsearch ''' if self.config['cafile'] != "": context = ssl.create_default_context(cafile=self.config['cafile']) else: context = ssl.create_default_context() context.check_hostname = self.config['check_hostname'] CONTEXT_VERIFY_MODES = { "none": ssl.CERT_NONE, "optional": ssl.CERT_OPTIONAL, "required": ssl.CERT_REQUIRED } context.verify_mode = CONTEXT_VERIFY_MODES[ self.config['cert_verification']] es_config = {'scheme': self.config['scheme'], 'ssl_context': context} if self.config['auth_method'] == 'api_key': es_config['api_key'] = self.credentials else: es_config['http_auth'] = self.credentials if 'distro' in self.config: if self.config['distro'] == 'opensearch': from opensearchpy import OpenSearch return OpenSearch(self.config['hosts'], **es_config) else: return Elasticsearch(self.config['hosts'], **es_config) else: return Elasticsearch(self.config['hosts'], **es_config)
def __init__(self, host="127.0.0.1", port=9200): """Create a OpenSearch client.""" super().__init__() self._error_container = {} self.user = current_app.config.get("OPENSEARCH_USER", "user") self.password = current_app.config.get("OPENSEARCH_PASSWORD", "pass") self.ssl = current_app.config.get("OPENSEARCH_SSL", False) self.verify = current_app.config.get("OPENSEARCH_VERIFY_CERTS", True) self.timeout = current_app.config.get("OPENSEARCH_TIMEOUT", 10) parameters = {} if self.ssl: parameters["use_ssl"] = self.ssl parameters["verify_certs"] = self.verify if self.user and self.password: parameters["http_auth"] = (self.user, self.password) if self.timeout: parameters["timeout"] = self.timeout self.client = OpenSearch([{"host": host, "port": port}], **parameters) self.import_counter = Counter() self.import_events = [] self._request_timeout = current_app.config.get( "TIMEOUT_FOR_EVENT_IMPORT", self.DEFAULT_EVENT_IMPORT_TIMEOUT)
async def create_opensearch_client() -> Optional[OpenSearch]: """ Create an OpenSearch client, connected to the configured OpenSearch server. :return: a connected OpenSearch client instance """ settings = get_settings() # Create the client with SSL/TLS enabled, but hostname verification disabled client = OpenSearch( hosts=[{ "host": settings.opensearch_server, "port": settings.opensearch_port }], http_compress=True, # enables gzip compression for request bodies http_auth=(settings.opensearch_user, settings.opensearch_password), use_ssl=True, verify_certs=settings.certificate_verify, ssl_assert_hostname=False, ssl_show_warn=False, ca_certs=settings.certificate_authority_path, ) logger.info("Created OpenSearch client") return client
def get_aes_client(self): service = "es" session = boto3.Session() credentials = session.get_credentials() region = session.region_name if credentials is not None: self.aws_auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service) else: click.secho( message= "Can not retrieve your AWS credentials, check your AWS config", fg="red", ) aes_client = OpenSearch( hosts=[self.endpoint], http_auth=self.aws_auth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, ) return aes_client
def __init__(self, host=settings.OPENSEARCH_HOST): protocol = settings.OPENSEARCH_PROTOCOL protocol_config = {} if protocol == "https": protocol_config = { "scheme": "https", "port": 443, "use_ssl": True, "verify_certs": settings.OPENSEARCH_VERIFY_CERTS, } if settings.IS_AWS: http_auth = ("supersurf", settings.OPENSEARCH_PASSWORD) else: http_auth = (None, None) self.client = OpenSearch([host], http_auth=http_auth, connection_class=RequestsHttpConnection, **protocol_config) self.index_nl = settings.OPENSEARCH_NL_INDEX self.index_en = settings.OPENSEARCH_EN_INDEX self.index_unk = settings.OPENSEARCH_UNK_INDEX self.languages = {"nl": self.index_nl, "en": self.index_en}
def get_search_client(conn, silent=False): """ Returns the Open Search client connected through port forwarding settings """ host = conn.config.open_search.host protocol_config = { "scheme": "https", "port": 443, "use_ssl": True, "verify_certs": True, } http_auth = ("supersurf", conn.config.secrets.opensearch.password,) es_client = OpenSearch( [host], http_auth=http_auth, connection_class=RequestsHttpConnection, **protocol_config ) # test if it works if not silent and not es_client.cat.health(request_timeout=30): raise ValueError('Credentials do not work for Open Search') return es_client
def make_opensearch(index, filters, queries=None, exclusion_filters=None, range_filters=None, prefix_filters=None, terms_filters=None, es_url='https://opensearch.lco.global'): """ Make an OpenSearch query Parameters ---------- index : str Name of index to search filters : list of dicts Each dict has a criterion for an OpenSearch "filter" queries : list of dicts Each dict has a "type" and "query" entry. The 'query' entry is a dict that has a criterion for an OpenSearch "query" exclusion_filters : list of dicts Each dict has a criterion for an OpenSearch "exclude" range_filters: list of dicts Each dict has a criterion an OpenSearch "range filter" prefix_filters: terms_filters: es_url : str URL of the OpenSearch host Returns ------- search : opensearch_dsl.Search The OpenSearch object """ if queries is None: queries = [] if exclusion_filters is None: exclusion_filters = [] if range_filters is None: range_filters = [] if terms_filters is None: terms_filters = [] if prefix_filters is None: prefix_filters = [] es = OpenSearch(es_url) s = Search(using=es, index=index) for f in filters: s = s.filter('term', **f) for f in terms_filters: s = s.filter('terms', **f) for f in range_filters: s = s.filter('range', **f) for f in prefix_filters: s = s.filter('prefix', **f) for f in exclusion_filters: s = s.exclude('term', **f) for q in queries: s = s.query(q['type'], **q['query']) return s
def setUpClass(cls): super().setUpClass() cls.search = OpenSearch( [settings.OPENSEARCH_HOST] ) cls.search.indices.create(settings.OPENSEARCH_NL_INDEX, ignore=400, body=cls.index_body('nl')) cls.search.indices.create(settings.OPENSEARCH_EN_INDEX, ignore=400, body=cls.index_body('en')) cls.search.indices.create(settings.OPENSEARCH_UNK_INDEX, ignore=400, body=cls.index_body('unk'))
def __get_es_client(self): if self.auth_type == OpensearchHandler.AuthType.NO_AUTH: if self._client is None: self._client = OpenSearch( hosts=self.hosts, use_ssl=self.use_ssl, verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, serializer=self.serializer) return self._client if self.auth_type == OpensearchHandler.AuthType.BASIC_AUTH: if self._client is None: return OpenSearch(hosts=self.hosts, http_auth=self.auth_details, use_ssl=self.use_ssl, verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, serializer=self.serializer) return self._client if self.auth_type == OpensearchHandler.AuthType.AWS_SIGNED_AUTH: if self.aws_session is None: raise ValueError( "AWS signed authentication enabled, but session object is None" ) if self._client is None: credentials = self.aws_session.get_credentials() awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, self.aws_session.region_name, 'es', session_token=credentials.token) self._client = OpenSearch( hosts=self.hosts, http_auth=awsauth, use_ssl=self.use_ssl, verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, serializer=self.serializer) return self._client raise ValueError("Authentication method not supported")
def get_search_db_connection(endpoint: str, region_name: str): """ Get a connection to an ElasticSearch or OpenSearch DB :param endpoint: cluster endpoint :param region_name: cluster region e.g. us-east-1 """ from opensearchpy import OpenSearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth verify_certs = False use_ssl = False # use ssl? if "https://" in endpoint: use_ssl = True # TODO remove this condition once ssl certs are available for .es.localhost.localstack.cloud domains endpoint_netloc = urlparse(endpoint).netloc if not re.match(r"^.*(localhost(\.localstack\.cloud)?)(:\d+)?$", endpoint_netloc): verify_certs = True LOG.debug("Creating ES client with endpoint %s", endpoint) if ENV_ACCESS_KEY in os.environ and ENV_SECRET_KEY in os.environ: access_key = os.environ.get(ENV_ACCESS_KEY) secret_key = os.environ.get(ENV_SECRET_KEY) session_token = os.environ.get(ENV_SESSION_TOKEN) awsauth = AWS4Auth(access_key, secret_key, region_name, "es", session_token=session_token) connection_class = RequestsHttpConnection return OpenSearch( hosts=[endpoint], verify_certs=verify_certs, use_ssl=use_ssl, connection_class=connection_class, http_auth=awsauth, ) return OpenSearch(hosts=[endpoint], verify_certs=verify_certs, use_ssl=use_ssl)
def create_es_conn(awsauth, es_hostname): es_conn = OpenSearch(hosts=[{ 'host': es_hostname, 'port': 443 }], http_auth=awsauth, use_ssl=True, http_compress=True, verify_certs=True, retry_on_timeout=True, connection_class=RequestsHttpConnection, timeout=60) return es_conn
def get_opensearch_client(self): ssl_context = self.ssl_context = create_ssl_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE opensearch_client = OpenSearch( [self.endpoint], http_auth=self.http_auth, verify_certs=False, ssl_context=ssl_context, connection_class=RequestsHttpConnection, ) return opensearch_client
def get_client(cls): """Returns an instantiated OpenSearch object. Caches result and returns cached object if already instantiated. If running on Production with the _VCAP_SERVICES environment variable, uses the bound OpenSearch service. If running locally, uses the OPENSEARCH_HOST and OPENSEARCH_PORT environment variables. """ if not cls._os_client: logger.info("Instantiating OpenSearch client") if settings.OPENSEARCH_URI: credentials = settings.OPENSEARCH_URI else: credentials = {"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT} cls._os_client = OpenSearch([credentials]) return cls._os_client
def __init__(self): host = 'localhost' port = 9200 auth = ('admin', 'admin') # certs = 'esnode.pem' # Elasticsearchインタンスの作成 self.es = OpenSearch( hosts=[{'host': host, 'port': port}], http_auth=auth, use_ssl=True, verify_certs=False, # ca_certs=certs, ssl_assert_hostname=False, ssl_show_warn=False, )
def __init__(self, auth, host='localhost', port=9200, index_name='book'): self.fingerprintCreator = FootPrintCreator() self.index_name = index_name self.client = OpenSearch( hosts=[{'host': host, 'port': port}], http_compress=True, # enables gzip compression for request bodies http_auth=auth, # client_cert = client_cert_path, # client_key = client_key_path, use_ssl=True, verify_certs=False, ssl_assert_hostname=False, ssl_show_warn=False, # ca_certs = ca_certs_path ) if not self.client.indices.exists(index=index_name): response = self.client.indices.create(index=index_name, body=bookMapping) print('\nCreating index:') print(response)
def test_es_search(): """Search after running management command to fill ES from data sources.""" es = OpenSearch([{"host": "localhost", "port": 9200}]) query = { "query": { "match": { "fi": { "query": "kivist", "fuzziness": "AUTO" } } } } s = es.search(index="test-index", body=query) hits = s["hits"]["total"]["value"] assert hits == 1
def set_connection(self, is_reconnect=False): urllib3.disable_warnings() logging.captureWarnings(True) if self.http_auth: opensearch_client = self.get_opensearch_client() elif self.use_aws_authentication: opensearch_client = self.get_aes_client() else: opensearch_client = OpenSearch([self.endpoint], verify_certs=True) # check connection. check OpenSearch SQL plugin availability. try: if not self.is_sql_plugin_installed(opensearch_client): click.secho( message= "Must have OpenSearch SQL plugin installed in your OpenSearch" "instance!\nCheck this out: https://github.com/opensearch-project/sql", fg="red", ) click.echo(self.plugins) sys.exit() # info() may throw ConnectionError, if connection fails to establish info = opensearch_client.info() self.opensearch_version = info["version"]["number"] self.client = opensearch_client self.get_indices() except ConnectionError as error: if is_reconnect: # re-throw error raise error else: click.secho(message="Can not connect to endpoint %s" % self.endpoint, fg="red") click.echo(repr(error)) sys.exit(0)
def get_search_client(): opensearch_url = settings.OPENSEARCH_HOST protocol = settings.OPENSEARCH_PROTOCOL protocol_config = {} if protocol == "https": protocol_config = { "scheme": "https", "port": 443, "use_ssl": True, "verify_certs": settings.OPENSEARCH_VERIFY_CERTS, } if settings.IS_AWS: http_auth = ("supersurf", settings.OPENSEARCH_PASSWORD) else: http_auth = (None, None) return OpenSearch([opensearch_url], http_auth=http_auth, connection_class=RequestsHttpConnection, **protocol_config)
output = "\n".join(output) click.echo(output) def pretty_print(s): try: d = json.loads(s) print(json.dumps(d, indent=2)) except json.decoder.JSONDecodeError: print(s) sql_cmd = DocTestConnection(query_language="sql") ppl_cmd = DocTestConnection(query_language="ppl") test_data_client = OpenSearch([ENDPOINT], verify_certs=True) def sql_cli_transform(s): return u'sql_cmd.process({0})'.format(repr(s.strip().rstrip(';'))) def ppl_cli_transform(s): return u'ppl_cmd.process({0})'.format(repr(s.strip().rstrip(';'))) def bash_transform(s): # TODO: add ppl support, be default cli uses sql if s.startswith("opensearchsql"): s = re.search(r"opensearchsql\s+-q\s+\"(.*?)\"", s).group(1) return u'cmd.process({0})'.format(repr(s.strip().rstrip(';')))
from opensearchpy import OpenSearch, NotFoundError import json import os import pprint INDEX_DOES_NOT_EXIST = 'index_not_found_exception' search_domain_scheme = os.environ.get('OPENSEARCH_DOMAIN_SCHEME', 'https') search_domain_host = os.environ['OPENSEARCH_DOMAIN_HOST'] search_domain_port = os.environ.get('OPENSEARCH_DOMAIN_PORT', 443) INDEX_PRODUCTS = 'products' search_client = OpenSearch( [search_domain_host], scheme=search_domain_scheme, port=search_domain_port, ) # -- Logging class LoggingMiddleware(object): def __init__(self, app): self._app = app def __call__(self, environ, resp): errorlog = environ['wsgi.errors'] pprint.pprint(('REQUEST', environ), stream=errorlog) def log_response(status, headers, *args): pprint.pprint(('RESPONSE', status, headers), stream=errorlog)
import os from flask import Flask from jinja2.tests import test_mapping, test_sequence from opensearchpy import OpenSearch app = Flask(__name__) app.config.from_object(os.getenv('CONFIGFILE', 'app.config-development')) #app.search = Elasticsearch(app.config.get('ELASTICSEARCH_NODES', ['localhost'])) opensearch_params = app.config.get( 'OPENSEARCH_PARAMS', { 'hosts': [{ 'host': 'localhost', 'port': 9200 }], 'http_auth': ('admin', 'admin'), 'use_ssl': True, 'verify_certs': False, 'ssl_show_warn': False, }) app.search = OpenSearch(**opensearch_params) from app.bp.tree import tree app.register_blueprint(tree, url_prefix='/tree') from app import routes
def index_products(event): # Conditionally creates and loads OpenSearch products index # If the products index already exists, this function does nothing. # Otherwise, this function will create the products index and add # all products from the bundled products.yaml file. search_domain_endpoint = event['ResourceProperties'][ 'OpenSearchDomainEndpoint'] logger.info('OpenSearch endpoint: %s', search_domain_endpoint) search_host = { 'host': search_domain_endpoint, 'port': 443, 'scheme': 'https', } # For testing: specify 'ForceIndex' to force existing index to be deleted and products indexed. force_index = event['ResourceProperties'].get( 'ForceIndex', 'no').lower() in ['true', 'yes', '1'] search = OpenSearch(hosts=[search_host], timeout=30, max_retries=10, retry_on_timeout=True) create_index_and_bulk_load = True if search.indices.exists(INDEX_NAME): logger.info('"%s" index already exists', INDEX_NAME) create_index_and_bulk_load = False if force_index: logger.info('Deleting index "%s"...', INDEX_NAME) res = search.indices.delete(index=INDEX_NAME) logger.debug("Delete index response: %s", res) create_index_and_bulk_load = True else: logger.info('Index does not exist') if create_index_and_bulk_load: request_body = { "settings": { "number_of_shards": 1, "number_of_replicas": 0 } } logger.info('Creating "%s" index...', INDEX_NAME) res = search.indices.create(index=INDEX_NAME, body=request_body) logger.debug("Create index response: %s", res) logger.info('Downloading products.yaml...') s3.meta.client.download_file(event['ResourceProperties']['Bucket'], event['ResourceProperties']['File'], '/tmp/products.yaml') with open('/tmp/products.yaml') as file: logger.info('Loading products.yaml...') products_list = yaml.safe_load(file) logger.info('Bulk indexing %s products in %s batches...', len(products_list), int(len(products_list) / MAX_BULK_BATCH_SIZE)) bulk_data = [] for product in products_list: bulk_data.append({ "index": { "_index": INDEX_NAME, "_type": TYPE_NAME, "_id": product[ID_FIELD] } }) bulk_data.append(product) if len(bulk_data) >= MAX_BULK_BATCH_SIZE: logger.debug("Indexing batch") search.bulk(index=INDEX_NAME, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: logger.debug("Indexing last batch") search.bulk(index=INDEX_NAME, body=bulk_data) logger.info('Products successfully indexed!')
if __name__ == '__main__': parser = ArgumentParser( description= 'import data from microcontroller_and_processors-2020-08-14.xlsx') parser.add_argument( 'infile', default='microcontroller_and_processors-2020-08-14.xlsx', help='input file') parser.add_argument('--tryout', '-t', action='store_true', help='output data instead of writing') args = parser.parse_args() df = pd.read_excel(args.infile) df_obj = df.select_dtypes(['object']) df[df_obj.columns] = df_obj.apply(clean_strings) df = df.T data = [clean_dict(df[d].to_dict()) for d in df] if args.tryout: from pprint import pprint pprint(data) else: ops = [{ '_op_type': 'index', '_index': 'mcs', '_type': 'document', '_source': set_vendor(m) } for m in data] osbulk(OpenSearch(**OPENSEARCH_PARAMS), ops)
# -*- coding: utf-8 -*- import argparse from opensearchpy import OpenSearch from salver.common.facts import all_facts client = OpenSearch(hosts=[{ "host": "localhost", "port": 9200 }], http_auth=('admin', 'admin'), use_ssl=True, verify_certs=False) replicas = 0 refresh_interval = "5s" def create_es_mappings(): for fact, body in all_facts.items(): index_name = f"salver-facts-{fact.lower()}-*" mapping = body.elastic_mapping() template = { "settings": { "index": { "number_of_shards": 2, "number_of_replicas": 1 } }, **mapping
def __init__(self, args): self.args = args self.filenames = set() self.osearch = OpenSearch(**OPENSEARCH_PARAMS) self._walk() self._read_files()
def __init__(self) -> None: if not getattr(self, "index_base_names", None): raise NotImplementedError( f"Importer {self.__class__.__name__} is missing index_base_names." ) self.es = OpenSearch([settings.ES_URI])
def test_es_basic_operations(): """Run basic operations for testing purposes.""" es = OpenSearch([{"host": "localhost", "port": 9200}]) try: logging.debug("Deleting existing test data") es.delete(index="unit-test-index", doc_type="test", id=1) except exceptions.NotFoundError: pass logging.debug("Adding test data") r = es.index( index="unit-test-index", doc_type="test", id=1, body={ "name": "Koira Koiruli Pöö", "height": "49", "mass": "10", "hair_color": "blond", "birth_year": "1999", "gender": "male", }, ) assert r["result"] == "created" es.indices.refresh(index="unit-test-index") r = es.get(index="unit-test-index", doc_type="test", id=1) assert r["_id"] == "1" s = es.search(index="unit-test-index", body={"query": { "match": { "name": "cat" } }}) hits = s["hits"]["total"]["value"] assert hits == 0 s = es.search(index="unit-test-index", body={"query": {"match_all": {}}}) logging.debug(s) hits = s["hits"]["total"]["value"] assert hits == 1 s = es.search(index="unit-test-index", body={"query": { "match": { "mass": "10" } }}) logging.debug(s) hits = s["hits"]["total"]["value"] assert hits == 1 s = es.search(index="unit-test-index", body={"query": { "match": { "name": "Koiruli" } }}) logging.debug(s) hits = s["hits"]["total"]["value"] assert hits == 1 logging.debug("Deleting test data") es.delete(index="unit-test-index", doc_type="test", id=1)
def connect( host: str, port: Optional[int] = 443, boto3_session: Optional[boto3.Session] = boto3.Session(), region: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, ) -> OpenSearch: """Create a secure connection to the specified Amazon OpenSearch domain. Note ---- We use `opensearch-py <https://github.com/opensearch-project/opensearch-py>`_, an OpenSearch python client. The username and password are mandatory if the OS Cluster uses `Fine Grained Access Control \ <https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html>`_. If fine grained access control is disabled, session access key and secret keys are used. Parameters ---------- host : str Amazon OpenSearch domain, for example: my-test-domain.us-east-1.es.amazonaws.com. port : int OpenSearch Service only accepts connections over port 80 (HTTP) or 443 (HTTPS) boto3_session : boto3.Session(), optional Boto3 Session. The default boto3 Session will be used if boto3_session receive None. region : AWS region of the Amazon OS domain. If not provided will be extracted from boto3_session. username : Fine-grained access control username. Mandatory if OS Cluster uses Fine Grained Access Control. password : Fine-grained access control password. Mandatory if OS Cluster uses Fine Grained Access Control. Returns ------- opensearchpy.OpenSearch OpenSearch low-level client. https://github.com/opensearch-project/opensearch-py/blob/main/opensearchpy/client/__init__.py """ valid_ports = {80, 443} if port not in valid_ports: raise ValueError(f"results: port must be one of {valid_ports}") if username and password: http_auth = (username, password) else: if region is None: region = _utils.get_region_from_session( boto3_session=boto3_session) creds = _utils.get_credentials_from_session( boto3_session=boto3_session) if creds.access_key is None or creds.secret_key is None: raise exceptions.InvalidArgument( "One of IAM Role or AWS ACCESS_KEY_ID and SECRET_ACCESS_KEY must be " "given. Unable to find ACCESS_KEY_ID and SECRET_ACCESS_KEY in boto3 " "session.") http_auth = AWS4Auth(creds.access_key, creds.secret_key, region, "es", session_token=creds.token) try: es = OpenSearch( host=_strip_endpoint(host), port=port, http_auth=http_auth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, timeout=30, max_retries=10, retry_on_timeout=True, ) except Exception as e: _logger.error( "Error connecting to Opensearch cluster. Please verify authentication details" ) raise e return es