def get_open_distro_client(self):
ssl_context = self.ssl_context = create_ssl_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
open_distro_client = OpenSearch(
[self.endpoint],
http_auth=self.http_auth,
verify_certs=False,
ssl_context=ssl_context,
connection_class=RequestsHttpConnection,
)
return open_distro_client
def get_aes_client(self):
service = "es"
session = boto3.Session()
credentials = session.get_credentials()
region = session.region_name
if credentials is not None:
self.aws_auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service)
else:
click.secho(message="Can not retrieve your AWS credentials, check your AWS config", fg="red")
aes_client = OpenSearch(
hosts=[self.endpoint],
http_auth=self.aws_auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
)
return aes_client
def set_connection(self, is_reconnect=False):
urllib3.disable_warnings()
logging.captureWarnings(True)
if self.http_auth:
opensearch_client = self.get_opensearch_client()
elif self.use_aws_authentication:
opensearch_client = self.get_aes_client()
else:
opensearch_client = OpenSearch([self.endpoint], verify_certs=True)
# check connection. check OpenSearch SQL plugin availability.
try:
if not self.is_sql_plugin_installed(opensearch_client):
click.secho(
message=
"Must have OpenSearch SQL plugin installed in your OpenSearch"
"instance!\nCheck this out: https://github.com/opensearch-project/sql",
fg="red",
)
click.echo(self.plugins)
sys.exit()
# info() may throw ConnectionError, if connection fails to establish
info = opensearch_client.info()
self.opensearch_version = info["version"]["number"]
self.client = opensearch_client
self.get_indices()
except ConnectionError as error:
if is_reconnect:
# re-throw error
raise error
else:
click.secho(message="Can not connect to endpoint %s" %
self.endpoint,
fg="red")
click.echo(repr(error))
sys.exit(0)
def main(argv=None):
'''
- TODO: read the directory from command line argument or use default location
- create new index with mappings and settings
- go through all the directories and read all html files and index it into this index
- TODO: point the alias to this new index
'''
args = parse_args(argv)
os_client = OpenSearch([SEARCH_ENDPOINT],
http_auth=(SEARCH_USER, SEARCH_PASS))
# print("Listing all indices : ")
# print(os_client.cat.indices())
# return
print("Creating a new index")
new_index = create_index_name_from_prefix()
status = os_client.indices.create(index=new_index,
body={
"settings": index_settings(),
"mappings": index_mappings()
})
print("Created a new index: ", new_index)
print(SECTION_SEPARATOR)
print("Listing all indices : ")
print(os_client.cat.indices())
print(SECTION_SEPARATOR)
d = os.getcwd()
print("fetching all html files under directory: ", d)
folder_name, doc_path, all_html_files = get_all_html_files_under_path(
'docs', path=d)
print("Total {} HTML files found".format(len(all_html_files)))
time.sleep(2)
try:
print("Starting bulk indexing to OpenSearch cluster..")
os_response = helpers.bulk(os_client,
yield_docs(all_html_files, doc_path,
folder_name, new_index),
chunk_size=5,
request_timeout=20)
print("\nhelpers.bulk() RESPONSE:", os_response)
print(SECTION_SEPARATOR)
except Exception as err:
print("\nhelpers.bulk() ERROR:", err)
print(SECTION_SEPARATOR)
sys.exit(1)
if os_client.indices.exists_alias(INDEX_ALIAS):
print("Indices associated with alias '{}' exists ".format(INDEX_ALIAS))
print(os_client.cat.aliases())
print(
"Deleting existing alias associated with '{}'".format(INDEX_ALIAS))
os_client.indices.delete_alias([INDEX_NAME_PREFIX + "*"], INDEX_ALIAS)
print(os_client.cat.aliases())
print("Updating alias {} to point to new index {}...".format(
INDEX_ALIAS, new_index))
os_client.indices.put_alias([new_index], INDEX_ALIAS)
print(os_client.cat.aliases())
output = "\n".join(output)
click.echo(output)
def pretty_print(s):
try:
d = json.loads(s)
print(json.dumps(d, indent=2))
except json.decoder.JSONDecodeError:
print(s)
sql_cmd = DocTestConnection(query_language="sql")
ppl_cmd = DocTestConnection(query_language="ppl")
test_data_client = OpenSearch([ENDPOINT], verify_certs=True)
def sql_cli_transform(s):
return u'sql_cmd.process({0})'.format(repr(s.strip().rstrip(';')))
def ppl_cli_transform(s):
return u'ppl_cmd.process({0})'.format(repr(s.strip().rstrip(';')))
def bash_transform(s):
# TODO: add ppl support, be default cli uses sql
if s.startswith("opensearchsql"):
s = re.search(r"opensearchsql\s+-q\s+\"(.*?)\"", s).group(1)
return u'cmd.process({0})'.format(repr(s.strip().rstrip(';')))