def results(): text = request.args.get("text") es = Elasticsearch([{'host': host, 'port': port}]) # now we need to search the es for {text} both in the {text} and the {title} # fields doc = { 'size': pagination_size, 'query': { "multi_match": { "query": text, "fields": ["text", "title"] } } } html = "<html><body>" html += "<br></br>" res = es.search(index=index, body=doc) number_of_results = res['hits']['total']['value'] hits = res['hits']['hits'] html += "there are {number_of_results} results...".format( number_of_results=number_of_results) html += "<br></br>" html + "<ul>" for hit in hits: f_text = hit['_source']['text'] f_title = hit['_source']['title'] html += "<li>{f_title}: {f_text}</li>".format(f_text=f_text, f_title=f_title) html += "<br></br>" html + "</ul>" html += "</body></html>" return html
def connect_to_elasticsearch(): from diskover import config global es_conn # Check if we are using AWS es if config['aws'] == "true": es_conn = Elasticsearch(hosts=config['es_host'], port=config['es_port'], use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, timeout=config['es_timeout'], maxsize=config['es_maxsize'], max_retries=config['es_max_retries'], retry_on_timeout=True) # Local connection to es else: es_conn = Elasticsearch(hosts=config['es_host'], port=config['es_port'], http_auth=(config['es_user'], config['es_password']), connection_class=Urllib3HttpConnection, timeout=config['es_timeout'], maxsize=config['es_maxsize'], max_retries=config['es_max_retries'], retry_on_timeout=True)
def _connect(self): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if len(self.config.ES_CERT_DIR) and os.path.isdir( self.config.ES_CERT_DIR): _LOGGER.warning( "Using cert and key in %s for connection to %s (verify_certs=%s)." % ( self.config.ES_CERT_DIR, self.config.ES_ENDPOINT, self.config.ES_VERIFY_CERTS, )) self.es = Elasticsearch( self.config.ES_ENDPOINT, use_ssl=self.config.ES_USE_SSL, verify_certs=self.config.ES_VERIFY_CERTS, client_cert=os.path.join(self.config.ES_CERT_DIR, "es.crt"), client_key=os.path.join(self.config.ES_CERT_DIR, "es.key"), timeout=60, max_retries=2, ) else: _LOGGER.warning( "Conecting to ElasticSearch without authentication.") self.es = Elasticsearch( self.config.ES_ENDPOINT, use_ssl=self.config.ES_USE_SSL, verify_certs=self.config.ES_VERIFY_CERTS, timeout=60, max_retries=2, )
def __init__(self): # self.ES = ["http://*****:*****@172.18.155.50:9200"] self.ES = ["192.168.31.146:9202"] # 创建es客户端 self.es = Elasticsearch( self.ES, # 启动前嗅探es集群服务器 sniff_on_start=True, # es集群服务器结点连接异常时是否刷新es结点信息 sniff_on_connection_fail=True, # 每60秒刷新节点信息 sniffer_timeout=80)
def es(self): """Connection to the ElasticSearch server.""" if not hasattr(self, "_es"): opts = dict(host=self.host, port=self.port, timeout=300) if self.auth: opts["http_auth"] = self.auth self._es = Elasticsearch([opts]) return self._es
def es(self): """Connection to the Elasticsearch server.""" if not hasattr(self, "_es"): opts = dict(host=self.host, port=self.port, timeout=300) auth = self.opts.get("auth") if auth: opts["http_auth"] = auth.split(",") self._es = Elasticsearch([opts]) return self._es
def handler(event, context): """ Entry point to the lambda function """ es_host = os.environ.get('es_host', 'localhost') es_port = os.environ.get('es_port', '9200') es_scheme = os.environ.get('es_scheme', 'http') es = Elasticsearch([{'host': es_host, 'port': es_port}], scheme=es_scheme) es.indices.create('member', body=MEMBER_INDEX, ignore=400) records = transform(event) bulk(es, records)
def get_test_client(nowait=False, **kwargs): # construct kwargs from the environment kw = {"timeout": 30, "ca_certs": ".ci/certs/ca.pem"} if "PYTHON_CONNECTION_CLASS" in os.environ: from elasticsearch7 import connection kw["connection_class"] = getattr(connection, os.environ["PYTHON_CONNECTION_CLASS"]) kw.update(kwargs) client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw) # wait for yellow status for _ in range(1 if nowait else 100): try: client.cluster.health(wait_for_status="yellow") return client except ConnectionError: time.sleep(0.1) else: # timeout raise SkipTest("Elasticsearch failed to start.")
class SinafinancespiderESPipeline(object): def __init__(self): # self.ES = ["http://*****:*****@172.18.155.50:9200"] self.ES = ["192.168.31.146:9202"] # 创建es客户端 self.es = Elasticsearch( self.ES, # 启动前嗅探es集群服务器 sniff_on_start=True, # es集群服务器结点连接异常时是否刷新es结点信息 sniff_on_connection_fail=True, # 每60秒刷新节点信息 sniffer_timeout=80) def process_item(self, item, spider): if item.collection == 'xmlyinfo': spider.logger.info("-----enter into insert ESXmlyInfo") doc = { "from_url": item["link"], "title": item["title"], "play_count": item["play_count"], "audio_time": item["duration"], "comment_count": item["commentCount"], "analyze_summary": item["introduction"], "author": item["author"], "add_time": item["addtime"], "create_time": item["addtime"], "release_time": item['times'], "id": item["id"] + str(int(time.time())), # "column": item['cat_name'], "source": "喜马拉雅", # "cover_url": item['cover_url'], "img_path": item['img_path'], "file_path": item['local_path'], "url_zj": item['url_zj'], "book_name": item['book_name'], "catagory": item['catagory'], "times": item['times'], # "trackId": item['trackId'] } ##新增加的xmlykeyinfo的写入 elif item.collection == 'xmlykeyinfo': spider.logger.info("-----enter into insert ESXmlyInfo") doc = { "from_url": item["link"], "title": item["title"], "play_count": item["play_count"], "comment_count": item["commentCount"], "analyze_summary": item["introduction"], "author": item["author"], "add_time": item["addtime"], "create_time": item["addtime"], "release_time": item['times'], "id": item["id"] + str(int(time.time())), "column": item['cat_name'], "source": "喜马拉雅", "cover_url": item['cover_url'], "img_path": item['img_path'], "file_path": item['local_path'], "url_zj": item['url_zj'], "book_name": item['book_name'], "catagory": item['catagory'], "times": item['times'], # "trackId": item['trackId'] } # self.es.index(index="bus_analysis_result", doc_type="sinafinance", body=doc, id=item["id"]) self.es.index(index="bus_analysis_result", body=doc, id=item["id"] + str(int(time.time()))) return item
def __init__(self, config, bgRun=True): self.config = config if "Elasticsearch" in config: esHost = config['Elasticsearch']['host'] esUser = config['Elasticsearch']['username'] esPass = config['Elasticsearch']['password'] esPort = config['Elasticsearch']['port'] self.es = Elasticsearch([esHost], http_auth=(esUser, esPass), ssl_show_warn=False, verify_certs=False, port=esPort) if "Notifications" in config: if "clicksendAPI_username" in config['Notifications']: configuration = clicksend_client.Configuration() configuration.username = config['Notifications'][ 'clicksendAPI_username'] configuration.password = config['Notifications'][ 'clicksendAPI_password'] self.notificationNumber = config['Notifications'][ 'phonenumber'] # create an instance of the API class self.smsAccountAPI = clicksend_client.AccountApi( clicksend_client.ApiClient(configuration)) try: # Get account information api_response = self.smsAccountAPI.account_get() if "http_code': 200" not in api_response: print("Invalid clicksend API response") print(api_response) self.smsAccountAPI = False else: self.smsAPI = clicksend_client.SMSApi( clicksend_client.ApiClient(configuration)) except ApiException as e: print( "Exception when calling AccountApi->account_get: %s\n" % e) except Exception as err: print("Eception when calling clicksend API") print(err) if "Wemo" in config: if "AlwaysOn" in config['Wemo']: print("Loading always on devices") self.alwaysOnDevices = config['Wemo']['AlwaysOn'].split(",") if self.es: print("Connecting to ES") try: if not self.es.ping(): print(self.es.ping()) raise ConnectionError( "Error connecting to Elasticsearch host: %s" % esHost) except Exception as err: print("Error connecting to ES") self.discovery() self.bgRun = bgRun if self.bgRun: self.bgUpdateThread = threading.Thread(target=self.update) self.bgUpdateThread.start()
#!/usr/bin/env python from elasticsearch7 import Elasticsearch import json import sys es = Elasticsearch([{'host': 'localhost', 'port': 9200}]) doc = {'size': 10000, 'query': {'match_all': {}}} res = es.search(index='my-index', body=doc, scroll='1m') number_of_results = res['hits']['total']['value'] print(f"number_of_results {number_of_results}") hits = res['hits']['hits'] for hit in hits: print(json.dumps(hit['_source'], indent=4, sort_keys=True))
#!/usr/bin/python3 from elasticsearch7 import Elasticsearch client = Elasticsearch("http://localhost:9200") resp = client.info() print(resp)
#!/usr/bin/python3 from elasticsearch7 import Elasticsearch from constants import index, host, port, filename import csv """ What will this script do? 1) check that es is running and try to connect to it. If not - show error to user. 2) check that es has an index called "index" if not, create it. 3) if we have an index called "index" - removed all data from it. 4) enter our set of data (notes) into the search engine. """ es = Elasticsearch([{'host': host, 'port': port}]) """ checking elasticsearch connection status as per: https://stackoverflow.com/questions/31208136/check-elasticsearch-connection-status-in-python/31644507 """ if not es.ping(): raise ValueError( f"cannot connect to es. Please make sure you have elastic search running on {host}:{port}" ) print(f"We connected to es") if es.indices.exists(index=index): print("yes index exists, removing data from the index") res = es.delete_by_query(index=index, body={"query": {"match_all": {}}}) deleted = res['deleted'] assert res['total'] == deleted print(f"deleted {deleted} records") else:
def get_es_client(): return Elasticsearch({'host': 'localhost', 'port': 9200}, timeout=300)
import json, glob, os from elasticsearch7 import Elasticsearch from datetime import datetime, timedelta from geoip import geolite2 es = Elasticsearch( [os.environ.get('ELASTIC_HOST')], http_auth=(os.environ.get('ELASTIC_USER'), os.environ.get('ELASTIC_PASSWORD')), scheme=os.environ.get('ELASTIC_PROTOCOL'), port=os.environ.get('ELASTIC_PORT'), ) ELASTIC_INDEX = "ptt" mappings = { "mappings": { "properties": { "geo_location": { "type": "geo_point" } } } } # ignore 400 index already exists es.indices.create(index=ELASTIC_INDEX, body=mappings, ignore=400) def index_data(file): with open(file) as f:
class Wemo: devices = [] cache_file = "wemo.cache" lastInsightUpdate = False insightDataCacheTime = 60 bgRun = True history = {"total_power": {"datetime": [], "data": []}, "devices": {}} historyMax = 360000 firstRun = True lastDiscoveryTime = datetime.datetime.now() rediscoverTries = 0 fakeData = True es = False config = False alwaysOnDevices = [] smsAccountAPI = False smsAPI = False notificationNumber = False def __init__(self, config, bgRun=True): self.config = config if "Elasticsearch" in config: esHost = config['Elasticsearch']['host'] esUser = config['Elasticsearch']['username'] esPass = config['Elasticsearch']['password'] esPort = config['Elasticsearch']['port'] self.es = Elasticsearch([esHost], http_auth=(esUser, esPass), ssl_show_warn=False, verify_certs=False, port=esPort) if "Notifications" in config: if "clicksendAPI_username" in config['Notifications']: configuration = clicksend_client.Configuration() configuration.username = config['Notifications'][ 'clicksendAPI_username'] configuration.password = config['Notifications'][ 'clicksendAPI_password'] self.notificationNumber = config['Notifications'][ 'phonenumber'] # create an instance of the API class self.smsAccountAPI = clicksend_client.AccountApi( clicksend_client.ApiClient(configuration)) try: # Get account information api_response = self.smsAccountAPI.account_get() if "http_code': 200" not in api_response: print("Invalid clicksend API response") print(api_response) self.smsAccountAPI = False else: self.smsAPI = clicksend_client.SMSApi( clicksend_client.ApiClient(configuration)) except ApiException as e: print( "Exception when calling AccountApi->account_get: %s\n" % e) except Exception as err: print("Eception when calling clicksend API") print(err) if "Wemo" in config: if "AlwaysOn" in config['Wemo']: print("Loading always on devices") self.alwaysOnDevices = config['Wemo']['AlwaysOn'].split(",") if self.es: print("Connecting to ES") try: if not self.es.ping(): print(self.es.ping()) raise ConnectionError( "Error connecting to Elasticsearch host: %s" % esHost) except Exception as err: print("Error connecting to ES") self.discovery() self.bgRun = bgRun if self.bgRun: self.bgUpdateThread = threading.Thread(target=self.update) self.bgUpdateThread.start() def discovery(self, retry=0): now = datetime.datetime.now() if self.firstRun: self.lastDiscoveryTime = now - datetime.timedelta(seconds=10000000) self.firstRun = False if (now - self.lastDiscoveryTime).seconds > 300 or retry > 0: self.lastDiscoveryTime = now print("Discovering Wemo devices on network") devicesA = pywemo.discover_devices() time.sleep(2) devicesB = pywemo.discover_devices() if len(devicesA) == len(devicesB) and len(devicesA) > 0: self.devices = devicesB print("Found %s devices in discovery" % len(self.devices)) else: if retry < 3: retry += 1 print( "Mismatch in number of detected devices, or no devices found. Trying again in 5 seconds." ) time.sleep(5) self.discovery(retry) else: print( "%s retries and still unable to get devices... backing off a long time (We will never give up though. The fridge depends on us.)" % retry) if retry > 10: time.sleep(120) else: time.sleep(30) retry += 1 self.discovery(retry) def sendSMSMessage(self, message, to, retry=0): if to.startswith("+1"): try: message = "Wemo Alert - %s " % message message = clicksend_client.SmsMessage(body=message, to=to) messages = clicksend_client.SmsMessageCollection( messages=[message]) try: # Send sms message(s) api_response = self.smsAPI.sms_send_post(messages) # print(api_response) except ApiException as e: print( "Exception when calling SMSApi->sms_send_post: %s\n" % e) except ConnectionResetError as err: print("Got an error sending SMS trying again...") time.sleep(1) if retry < 3: retry += 1 self.sendSMSMessage(message, to, retry) else: print( "Still couldn't send that dang message. Giving up after 3 retries" ) else: print("Invalid phone number while trying to send message") def reDiscover(self, rediscoveryTime=600): now = datetime.datetime.now() if (now - self.lastDiscoveryTime).seconds > rediscoveryTime: self.discovery() def writeInfotoES(self, infoData): # print(infoData['datetime']) try: res = self.es.index( index="wemo-%s-%s-%s" % (infoData['name'].replace(" ", "").lower(), infoData['macaddress'].replace(":", "").lower(), infoData['datetime'].strftime('%Y-%m-%d')), body=infoData) if "result" in res: if res['result'] == "created": return True return False except Exception as err: print("Caught exception while trying to save to ES") print(err) return False # print(res) def collectDeviceInfo(self, historyLimit=60): now = datetime.datetime.utcnow() info = {"devices": []} for device in self.devices: deviceCommunicating = True try: state = device.get_state(force_update=True) except ActionException as err: print(err) deviceCommunicating = False if deviceCommunicating: try: device.update_insight_params() except Exception as err: print("Error getting device (%s) update." % device.name) continue if state == 8: state = "Standby" elif state == 1: state = "On" elif state == 0: state = "off" data = { "name": device.name, "datetime": now, 'macaddress': device.mac, "status": state, "ontoday": device.today_on_time, "todaykwh": round(device.today_kwh, 2), "currentPower": device.current_power, "todayOnTime": device.today_on_time, "onFor": device.on_for, "todayStandbyTime": device.today_standby_time } info['devices'].append(data) if self.es: self.writeInfotoES(data) else: print("ERROR COMMUNICATING WITH %s. Re-running discovery." % device.name) if device.name in self.alwaysOnDevices: self.sendSMSMessage( "Error communicating with %s. Re-running discovery." % device.name, self.notificationNumber) print("OH GOD ITS A BAD ONE TO LOSE.") self.discovery() return info def print_power_data(self): self.updateInsight() for device in self.devices: print(device.name) print(device.current_power) def print_total_power(self): print("Total power (kw): %s" % (self.total_power() / 1000000)) def alwaysOnDevice(self, device, flipped=0): if device.name in self.alwaysOnDevices: if device.get_state() == 0: self.sendSMSMessage( "%s is off! Turning it back on!" % device.name, self.notificationNumber) print("%s is off! Turning it back on!" % device.name) device.on() time.sleep(1) if device.get_state(True) == 0: if flipped <= 3: flipped += 1 time.sleep(1) self.alwaysOnDevice(device, flipped=flipped) else: self.sendSMSMessage( "%s is still off. Unable to turn it back on!" % device.name, self.notificationNumber) print("%s is still off. Unable to turn it back on!" % device.name) def checkAlwaysOn(self): for device in self.devices: self.alwaysOnDevice(device) for device in self.alwaysOnDevices: if not any(x.name == device for x in self.devices): try: print("OH MY GOD %s IS MISSING!" % device) self.sendSMSMessage( "%s not found in device list. Re-running discovery." % device.name, self.notificationNumber) self.discovery() except AttributeError as err: print( "Weird error when trying to check always on. Device likely isn't a device?" ) print(err) self.discovery() def update(self): while self.bgRun: self.reDiscover() self.collectDeviceInfo() self.checkAlwaysOn() time.sleep(30)
action='store_false', dest='execute_watch') parser.set_defaults(endpoint='localhost', port="9200", protocol="http", test_file='data.json', user='******', password='******') args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) es = Elasticsearch( [args.protocol + "://" + args.endpoint + ":" + args.port], http_auth=(args.user, args.password), ca_certs=args.cacert) test = load_file(args.test_file) if args.test_index: # Load Mapping try: es.indices.delete(test['index']) except Exception as err: print("Unable to delete current dataset") pass index_template = load_file(test['mapping_file']) for unneeded_keys in ['order', 'version', 'index_patterns']: index_template.pop(unneeded_keys, None) es.indices.create(index=test["index"], body=index_template)
def test_handler(): """ Test the service handler """ es = Elasticsearch() es.indices.delete('member', ignore_unavailable=True) service.handler( { "Records": [{ "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", "body": json.dumps({ "_id": "abc", "firstName": "John", "lastName": "Doe", "hashedEmail": "0bc83cb571cd1c50ba6f3e8a78ef1346", "email": "*****@*****.**" }) }, { "messageId": "2e1424d4-f796-459a-8184-9c92662be6da", "body": json.dumps({ "_id": "def", "firstName": "Jane", "lastName": "River", "isPublic": True, "interests": ["Volley Ball", "Natation"], "virtualStudies": [{ "_id": "1", "name": "Virtual Study 1" }, { "_id": "12", "name": "Virtual Study 2" }] }), }] }, {}) a = es.get('member', 'abc') assert a.get('found') assert a.get('_id') == 'abc' assert a['_source'].get('firstName') == 'John' assert a['_source'].get('lastName') == 'Doe' assert not a['_source'].get('isPublic') assert a['_source'].get( 'hashedEmail') == '0bc83cb571cd1c50ba6f3e8a78ef1346' assert a['_source'].get('email') == '*****@*****.**' b = es.get('member', 'def') assert b.get('found') assert b.get('_id') == 'def' assert b['_source'].get('firstName') == 'Jane' assert b['_source'].get('lastName') == 'River' assert b['_source'].get('isPublic') assert b['_source'].get('interests') == ["Volley Ball", "Natation"] assert b['_source'].get('virtualStudies') == [{ "_id": "1", "name": "Virtual Study 1" }, { "_id": "12", "name": "Virtual Study 2" }] assert b['_source'].get('searchableInterests') == [{ "name": "Volley Ball" }, { "name": "Natation" }]
# coding:utf-8 import json from elasticsearch7 import Elasticsearch # /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/ from MooseSpider.items import ShiCiItem ES_URL = ['127.0.0.1:9200'] es = Elasticsearch(ES_URL) class Article: name = "" author = "" def test_ping(): pong = es.ping() print(pong) def test_delete(index): if not es.indices.exists(index): return False else: res = es.indices.delete(index=index) return res['acknowledged']
#!/usr/bin/python3 from elasticsearch7 import Elasticsearch import json import sys from constants import index, host, port es = Elasticsearch([{'host': host, 'port': port}]) doc = {'size': 10000, 'query': {'match_all': {}}} res = es.search(index=index, body=doc, scroll='1m') number_of_results = res['hits']['total']['value'] print(f"number_of_results {number_of_results}") hits = res['hits']['hits'] for hit in hits: print(json.dumps(hit['_source'], indent=4, sort_keys=True))
def all(): es = Elasticsearch([{'host': host, 'port': port}]) es.indices.refresh(index) res = es.cat.count(index, params={"format": "json"}) count = res[0]["count"] return form.format(count=count)
from functions.setup_elastic_search import reload_es print('Starting up...') reboot = "LuaUPnP boot" lastLine = "" appendedLine = "" vera_host = os.environ.get('VERA_HOST') es_host = os.environ['ES_HOST'] es_port = os.environ['ES_PORT'] vera_log_index = os.environ['VERA_LOG_INDEX'] sleep_time = int(os.environ['SLEEP_TIME']) skip_reload = os.environ['SKIP_RELOAD'] == "true" startup = True es = Elasticsearch(hosts=[{'host': es_host, 'port': es_port}]) url = "http://{}/cgi-bin/cmh/log.sh?Device=LuaUPnP".format(vera_host) es_base_url = "http://{}:{}".format(es_host, es_port) tag_line_exp = re.compile( r'\d{2}[\t]\d{2}[-/]\d{2}[-/]\d{2} \d{1,2}:\d{2}:\d{2}.\d{3}') date_exp = re.compile(r'\d{2}[-/]\d{2}[-/]\d{2} \d{1,2}:\d{2}:\d{2}.\d{3}') index_tag = re.compile(r'^<.{10}>$') if not skip_reload: reload_es(es_base_url) print("Elastic Search index {} has been deleted and reset...".format(vera_log_index)) def compose(message, timestamp, log_level): doc = { 'timestamp': convert_to_utc(timestamp), 'message': message,
""" Python 操作ES7 实现增删改查功能 """ # elasticsearch集群服务器的地址 ES = [ '39.98.152.206:9200', '39.98.152.206:9201' ] # 创建elasticsearch客户端 es = Elasticsearch( ES, # 启动前嗅探es集群服务器 sniff_on_start=True, # es集群服务器结点连接异常时是否刷新es节点信息 sniff_on_connection_fail=True, # 每60秒刷新节点信息 sniffer_timeout=60 ) ## 查询 query = { "query": { "match_all": {} } } ## 返回的是字典 ret_query = es.search(index='item', body=query) print(ret_query['hits'])
import re from elasticsearch7 import Elasticsearch from flask import Flask, render_template, request from pymongo import MongoClient import config elastic = Elasticsearch(config.ELASTIC_URL) # Process elasticsearch hits and return flights records def process_search(results): records = [] total = 0 if results["hits"] and results["hits"]["hits"]: total = results["hits"]["total"] hits = results["hits"]["hits"] for hit in hits: record = hit["_source"] records.append(record) return records, total # Calculate offsets for fetching lists of flights from MongoDB def get_navigation_offsets(offset1, offset2, increment): offsets = {} offsets["Previous"] = { "top_offset": max(offset2 - increment, 0), "bottom_offset": max(offset1 - increment, 0),
#!/usr/bin/env python from elasticsearch7 import Elasticsearch import random import time es = Elasticsearch([{'host': 'localhost', 'port': 9200}]) index_name = "my-index" day = 1 while True: elem = {"day": day, "speed_of_load": random.uniform(0, 1) * 3} res = es.index(index=index_name, body=elem) assert res['_shards']['successful'] == 1 day += 1 print("added another record...") time.sleep(1) print(f"inserted {len(data_elements)} records")
) parser.add_argument('--user', help='user') parser.add_argument('--password', help='password') parser.add_argument('--endpoint', help='endpoint') parser.add_argument('--port', help='port') parser.add_argument('--protocol', help='protocol') parser.add_argument('--test_file', help='test file') parser.set_defaults(endpoint='localhost', port="9200", protocol="http", test_file='data.json', user='******', password='******') args = parser.parse_args() es = Elasticsearch([args.protocol + "://" + args.endpoint + ":" + args.port], http_auth=(args.user, args.password)) def find_item(list, key): for item in list: if key in item: return item return None with open(args.test_file, 'r') as test_file: test = json.loads(test_file.read()) try: es.indices.delete(test['index']) except: print("Unable to delete current dataset")
args = vars(get_args()) if not args['comparecsvs']: if not args['index'] or not args['rootdir']: print( '--eshost1, --index and --rootdir cli args required (unless using --comparecsvs), use -h for help' ) sys.exit(1) # set up elasticsearch connections es = Elasticsearch(hosts=args['eshost1'], port=args['esport1'], http_auth=(args['esuser1'], args['espass1']), connection_class=Urllib3HttpConnection, timeout=config['es_timeout'], maxsize=config['es_maxsize'], max_retries=config['es_max_retries'], retry_on_timeout=True) if args['eshost2']: es2 = Elasticsearch(hosts=args['eshost2'], port=args['esport2'], http_auth=(args['esuser2'], args['espass2']), connection_class=Urllib3HttpConnection, timeout=config['es_timeout'], maxsize=config['es_maxsize'], max_retries=config['es_max_retries'], retry_on_timeout=True) else: es2 = es