def results():
text = request.args.get("text")
es = Elasticsearch([{'host': host, 'port': port}])
# now we need to search the es for {text} both in the {text} and the {title}
# fields
doc = {
'size': pagination_size,
'query': {
"multi_match": {
"query": text,
"fields": ["text", "title"]
}
}
}
html = "<html><body>"
html += "<br></br>"
res = es.search(index=index, body=doc)
number_of_results = res['hits']['total']['value']
hits = res['hits']['hits']
html += "there are {number_of_results} results...".format(
number_of_results=number_of_results)
html += "<br></br>"
html + "<ul>"
for hit in hits:
f_text = hit['_source']['text']
f_title = hit['_source']['title']
html += "<li>{f_title}: {f_text}</li>".format(f_text=f_text,
f_title=f_title)
html += "<br></br>"
html + "</ul>"
html += "</body></html>"
return html
def connect_to_elasticsearch():
from diskover import config
global es_conn
# Check if we are using AWS es
if config['aws'] == "true":
es_conn = Elasticsearch(hosts=config['es_host'],
port=config['es_port'],
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
timeout=config['es_timeout'],
maxsize=config['es_maxsize'],
max_retries=config['es_max_retries'],
retry_on_timeout=True)
# Local connection to es
else:
es_conn = Elasticsearch(hosts=config['es_host'],
port=config['es_port'],
http_auth=(config['es_user'],
config['es_password']),
connection_class=Urllib3HttpConnection,
timeout=config['es_timeout'],
maxsize=config['es_maxsize'],
max_retries=config['es_max_retries'],
retry_on_timeout=True)
def _connect(self):
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
if len(self.config.ES_CERT_DIR) and os.path.isdir(
self.config.ES_CERT_DIR):
_LOGGER.warning(
"Using cert and key in %s for connection to %s (verify_certs=%s)."
% (
self.config.ES_CERT_DIR,
self.config.ES_ENDPOINT,
self.config.ES_VERIFY_CERTS,
))
self.es = Elasticsearch(
self.config.ES_ENDPOINT,
use_ssl=self.config.ES_USE_SSL,
verify_certs=self.config.ES_VERIFY_CERTS,
client_cert=os.path.join(self.config.ES_CERT_DIR, "es.crt"),
client_key=os.path.join(self.config.ES_CERT_DIR, "es.key"),
timeout=60,
max_retries=2,
)
else:
_LOGGER.warning(
"Conecting to ElasticSearch without authentication.")
self.es = Elasticsearch(
self.config.ES_ENDPOINT,
use_ssl=self.config.ES_USE_SSL,
verify_certs=self.config.ES_VERIFY_CERTS,
timeout=60,
max_retries=2,
)
def __init__(self):
# self.ES = ["http://*****:*****@172.18.155.50:9200"]
self.ES = ["192.168.31.146:9202"]
# 创建es客户端
self.es = Elasticsearch(
self.ES,
# 启动前嗅探es集群服务器
sniff_on_start=True,
# es集群服务器结点连接异常时是否刷新es结点信息
sniff_on_connection_fail=True,
# 每60秒刷新节点信息
sniffer_timeout=80)
def es(self):
"""Connection to the ElasticSearch server."""
if not hasattr(self, "_es"):
opts = dict(host=self.host, port=self.port, timeout=300)
if self.auth:
opts["http_auth"] = self.auth
self._es = Elasticsearch([opts])
return self._es
def es(self):
"""Connection to the Elasticsearch server."""
if not hasattr(self, "_es"):
opts = dict(host=self.host, port=self.port, timeout=300)
auth = self.opts.get("auth")
if auth:
opts["http_auth"] = auth.split(",")
self._es = Elasticsearch([opts])
return self._es
def handler(event, context):
"""
Entry point to the lambda function
"""
es_host = os.environ.get('es_host', 'localhost')
es_port = os.environ.get('es_port', '9200')
es_scheme = os.environ.get('es_scheme', 'http')
es = Elasticsearch([{'host': es_host, 'port': es_port}], scheme=es_scheme)
es.indices.create('member', body=MEMBER_INDEX, ignore=400)
records = transform(event)
bulk(es, records)
def get_test_client(nowait=False, **kwargs):
# construct kwargs from the environment
kw = {"timeout": 30, "ca_certs": ".ci/certs/ca.pem"}
if "PYTHON_CONNECTION_CLASS" in os.environ:
from elasticsearch7 import connection
kw["connection_class"] = getattr(connection,
os.environ["PYTHON_CONNECTION_CLASS"])
kw.update(kwargs)
client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw)
# wait for yellow status
for _ in range(1 if nowait else 100):
try:
client.cluster.health(wait_for_status="yellow")
return client
except ConnectionError:
time.sleep(0.1)
else:
# timeout
raise SkipTest("Elasticsearch failed to start.")
class SinafinancespiderESPipeline(object):
def __init__(self):
# self.ES = ["http://*****:*****@172.18.155.50:9200"]
self.ES = ["192.168.31.146:9202"]
# 创建es客户端
self.es = Elasticsearch(
self.ES,
# 启动前嗅探es集群服务器
sniff_on_start=True,
# es集群服务器结点连接异常时是否刷新es结点信息
sniff_on_connection_fail=True,
# 每60秒刷新节点信息
sniffer_timeout=80)
def process_item(self, item, spider):
if item.collection == 'xmlyinfo':
spider.logger.info("-----enter into insert ESXmlyInfo")
doc = {
"from_url": item["link"],
"title": item["title"],
"play_count": item["play_count"],
"audio_time": item["duration"],
"comment_count": item["commentCount"],
"analyze_summary": item["introduction"],
"author": item["author"],
"add_time": item["addtime"],
"create_time": item["addtime"],
"release_time": item['times'],
"id": item["id"] + str(int(time.time())),
# "column": item['cat_name'],
"source": "喜马拉雅",
# "cover_url": item['cover_url'],
"img_path": item['img_path'],
"file_path": item['local_path'],
"url_zj": item['url_zj'],
"book_name": item['book_name'],
"catagory": item['catagory'],
"times": item['times'],
# "trackId": item['trackId']
}
##新增加的xmlykeyinfo的写入
elif item.collection == 'xmlykeyinfo':
spider.logger.info("-----enter into insert ESXmlyInfo")
doc = {
"from_url": item["link"],
"title": item["title"],
"play_count": item["play_count"],
"comment_count": item["commentCount"],
"analyze_summary": item["introduction"],
"author": item["author"],
"add_time": item["addtime"],
"create_time": item["addtime"],
"release_time": item['times'],
"id": item["id"] + str(int(time.time())),
"column": item['cat_name'],
"source": "喜马拉雅",
"cover_url": item['cover_url'],
"img_path": item['img_path'],
"file_path": item['local_path'],
"url_zj": item['url_zj'],
"book_name": item['book_name'],
"catagory": item['catagory'],
"times": item['times'],
# "trackId": item['trackId']
}
# self.es.index(index="bus_analysis_result", doc_type="sinafinance", body=doc, id=item["id"])
self.es.index(index="bus_analysis_result",
body=doc,
id=item["id"] + str(int(time.time())))
return item
def __init__(self, config, bgRun=True):
self.config = config
if "Elasticsearch" in config:
esHost = config['Elasticsearch']['host']
esUser = config['Elasticsearch']['username']
esPass = config['Elasticsearch']['password']
esPort = config['Elasticsearch']['port']
self.es = Elasticsearch([esHost],
http_auth=(esUser, esPass),
ssl_show_warn=False,
verify_certs=False,
port=esPort)
if "Notifications" in config:
if "clicksendAPI_username" in config['Notifications']:
configuration = clicksend_client.Configuration()
configuration.username = config['Notifications'][
'clicksendAPI_username']
configuration.password = config['Notifications'][
'clicksendAPI_password']
self.notificationNumber = config['Notifications'][
'phonenumber']
# create an instance of the API class
self.smsAccountAPI = clicksend_client.AccountApi(
clicksend_client.ApiClient(configuration))
try:
# Get account information
api_response = self.smsAccountAPI.account_get()
if "http_code': 200" not in api_response:
print("Invalid clicksend API response")
print(api_response)
self.smsAccountAPI = False
else:
self.smsAPI = clicksend_client.SMSApi(
clicksend_client.ApiClient(configuration))
except ApiException as e:
print(
"Exception when calling AccountApi->account_get: %s\n"
% e)
except Exception as err:
print("Eception when calling clicksend API")
print(err)
if "Wemo" in config:
if "AlwaysOn" in config['Wemo']:
print("Loading always on devices")
self.alwaysOnDevices = config['Wemo']['AlwaysOn'].split(",")
if self.es:
print("Connecting to ES")
try:
if not self.es.ping():
print(self.es.ping())
raise ConnectionError(
"Error connecting to Elasticsearch host: %s" % esHost)
except Exception as err:
print("Error connecting to ES")
self.discovery()
self.bgRun = bgRun
if self.bgRun:
self.bgUpdateThread = threading.Thread(target=self.update)
self.bgUpdateThread.start()
#!/usr/bin/env python
from elasticsearch7 import Elasticsearch
import json
import sys
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
doc = {'size': 10000, 'query': {'match_all': {}}}
res = es.search(index='my-index', body=doc, scroll='1m')
number_of_results = res['hits']['total']['value']
print(f"number_of_results {number_of_results}")
hits = res['hits']['hits']
for hit in hits:
print(json.dumps(hit['_source'], indent=4, sort_keys=True))
#!/usr/bin/python3
from elasticsearch7 import Elasticsearch
client = Elasticsearch("http://localhost:9200")
resp = client.info()
print(resp)
#!/usr/bin/python3
from elasticsearch7 import Elasticsearch
from constants import index, host, port, filename
import csv
"""
What will this script do?
1) check that es is running and try to connect to it.
If not - show error to user.
2) check that es has an index called "index" if not, create it.
3) if we have an index called "index" - removed all data from it.
4) enter our set of data (notes) into the search engine.
"""
es = Elasticsearch([{'host': host, 'port': port}])
""" checking elasticsearch connection status as per:
https://stackoverflow.com/questions/31208136/check-elasticsearch-connection-status-in-python/31644507
"""
if not es.ping():
raise ValueError(
f"cannot connect to es. Please make sure you have elastic search running on {host}:{port}"
)
print(f"We connected to es")
if es.indices.exists(index=index):
print("yes index exists, removing data from the index")
res = es.delete_by_query(index=index, body={"query": {"match_all": {}}})
deleted = res['deleted']
assert res['total'] == deleted
print(f"deleted {deleted} records")
else:
def get_es_client():
return Elasticsearch({'host': 'localhost', 'port': 9200}, timeout=300)
import json, glob, os
from elasticsearch7 import Elasticsearch
from datetime import datetime, timedelta
from geoip import geolite2
es = Elasticsearch(
[os.environ.get('ELASTIC_HOST')],
http_auth=(os.environ.get('ELASTIC_USER'),
os.environ.get('ELASTIC_PASSWORD')),
scheme=os.environ.get('ELASTIC_PROTOCOL'),
port=os.environ.get('ELASTIC_PORT'),
)
ELASTIC_INDEX = "ptt"
mappings = {
"mappings": {
"properties": {
"geo_location": {
"type": "geo_point"
}
}
}
}
# ignore 400 index already exists
es.indices.create(index=ELASTIC_INDEX, body=mappings, ignore=400)
def index_data(file):
with open(file) as f:
class Wemo:
devices = []
cache_file = "wemo.cache"
lastInsightUpdate = False
insightDataCacheTime = 60
bgRun = True
history = {"total_power": {"datetime": [], "data": []}, "devices": {}}
historyMax = 360000
firstRun = True
lastDiscoveryTime = datetime.datetime.now()
rediscoverTries = 0
fakeData = True
es = False
config = False
alwaysOnDevices = []
smsAccountAPI = False
smsAPI = False
notificationNumber = False
def __init__(self, config, bgRun=True):
self.config = config
if "Elasticsearch" in config:
esHost = config['Elasticsearch']['host']
esUser = config['Elasticsearch']['username']
esPass = config['Elasticsearch']['password']
esPort = config['Elasticsearch']['port']
self.es = Elasticsearch([esHost],
http_auth=(esUser, esPass),
ssl_show_warn=False,
verify_certs=False,
port=esPort)
if "Notifications" in config:
if "clicksendAPI_username" in config['Notifications']:
configuration = clicksend_client.Configuration()
configuration.username = config['Notifications'][
'clicksendAPI_username']
configuration.password = config['Notifications'][
'clicksendAPI_password']
self.notificationNumber = config['Notifications'][
'phonenumber']
# create an instance of the API class
self.smsAccountAPI = clicksend_client.AccountApi(
clicksend_client.ApiClient(configuration))
try:
# Get account information
api_response = self.smsAccountAPI.account_get()
if "http_code': 200" not in api_response:
print("Invalid clicksend API response")
print(api_response)
self.smsAccountAPI = False
else:
self.smsAPI = clicksend_client.SMSApi(
clicksend_client.ApiClient(configuration))
except ApiException as e:
print(
"Exception when calling AccountApi->account_get: %s\n"
% e)
except Exception as err:
print("Eception when calling clicksend API")
print(err)
if "Wemo" in config:
if "AlwaysOn" in config['Wemo']:
print("Loading always on devices")
self.alwaysOnDevices = config['Wemo']['AlwaysOn'].split(",")
if self.es:
print("Connecting to ES")
try:
if not self.es.ping():
print(self.es.ping())
raise ConnectionError(
"Error connecting to Elasticsearch host: %s" % esHost)
except Exception as err:
print("Error connecting to ES")
self.discovery()
self.bgRun = bgRun
if self.bgRun:
self.bgUpdateThread = threading.Thread(target=self.update)
self.bgUpdateThread.start()
def discovery(self, retry=0):
now = datetime.datetime.now()
if self.firstRun:
self.lastDiscoveryTime = now - datetime.timedelta(seconds=10000000)
self.firstRun = False
if (now - self.lastDiscoveryTime).seconds > 300 or retry > 0:
self.lastDiscoveryTime = now
print("Discovering Wemo devices on network")
devicesA = pywemo.discover_devices()
time.sleep(2)
devicesB = pywemo.discover_devices()
if len(devicesA) == len(devicesB) and len(devicesA) > 0:
self.devices = devicesB
print("Found %s devices in discovery" % len(self.devices))
else:
if retry < 3:
retry += 1
print(
"Mismatch in number of detected devices, or no devices found. Trying again in 5 seconds."
)
time.sleep(5)
self.discovery(retry)
else:
print(
"%s retries and still unable to get devices... backing off a long time (We will never give up though. The fridge depends on us.)"
% retry)
if retry > 10:
time.sleep(120)
else:
time.sleep(30)
retry += 1
self.discovery(retry)
def sendSMSMessage(self, message, to, retry=0):
if to.startswith("+1"):
try:
message = "Wemo Alert - %s " % message
message = clicksend_client.SmsMessage(body=message, to=to)
messages = clicksend_client.SmsMessageCollection(
messages=[message])
try:
# Send sms message(s)
api_response = self.smsAPI.sms_send_post(messages)
# print(api_response)
except ApiException as e:
print(
"Exception when calling SMSApi->sms_send_post: %s\n" %
e)
except ConnectionResetError as err:
print("Got an error sending SMS trying again...")
time.sleep(1)
if retry < 3:
retry += 1
self.sendSMSMessage(message, to, retry)
else:
print(
"Still couldn't send that dang message. Giving up after 3 retries"
)
else:
print("Invalid phone number while trying to send message")
def reDiscover(self, rediscoveryTime=600):
now = datetime.datetime.now()
if (now - self.lastDiscoveryTime).seconds > rediscoveryTime:
self.discovery()
def writeInfotoES(self, infoData):
# print(infoData['datetime'])
try:
res = self.es.index(
index="wemo-%s-%s-%s" %
(infoData['name'].replace(" ", "").lower(),
infoData['macaddress'].replace(":", "").lower(),
infoData['datetime'].strftime('%Y-%m-%d')),
body=infoData)
if "result" in res:
if res['result'] == "created":
return True
return False
except Exception as err:
print("Caught exception while trying to save to ES")
print(err)
return False
# print(res)
def collectDeviceInfo(self, historyLimit=60):
now = datetime.datetime.utcnow()
info = {"devices": []}
for device in self.devices:
deviceCommunicating = True
try:
state = device.get_state(force_update=True)
except ActionException as err:
print(err)
deviceCommunicating = False
if deviceCommunicating:
try:
device.update_insight_params()
except Exception as err:
print("Error getting device (%s) update." % device.name)
continue
if state == 8:
state = "Standby"
elif state == 1:
state = "On"
elif state == 0:
state = "off"
data = {
"name": device.name,
"datetime": now,
'macaddress': device.mac,
"status": state,
"ontoday": device.today_on_time,
"todaykwh": round(device.today_kwh, 2),
"currentPower": device.current_power,
"todayOnTime": device.today_on_time,
"onFor": device.on_for,
"todayStandbyTime": device.today_standby_time
}
info['devices'].append(data)
if self.es:
self.writeInfotoES(data)
else:
print("ERROR COMMUNICATING WITH %s. Re-running discovery." %
device.name)
if device.name in self.alwaysOnDevices:
self.sendSMSMessage(
"Error communicating with %s. Re-running discovery." %
device.name, self.notificationNumber)
print("OH GOD ITS A BAD ONE TO LOSE.")
self.discovery()
return info
def print_power_data(self):
self.updateInsight()
for device in self.devices:
print(device.name)
print(device.current_power)
def print_total_power(self):
print("Total power (kw): %s" % (self.total_power() / 1000000))
def alwaysOnDevice(self, device, flipped=0):
if device.name in self.alwaysOnDevices:
if device.get_state() == 0:
self.sendSMSMessage(
"%s is off! Turning it back on!" % device.name,
self.notificationNumber)
print("%s is off! Turning it back on!" % device.name)
device.on()
time.sleep(1)
if device.get_state(True) == 0:
if flipped <= 3:
flipped += 1
time.sleep(1)
self.alwaysOnDevice(device, flipped=flipped)
else:
self.sendSMSMessage(
"%s is still off. Unable to turn it back on!" %
device.name, self.notificationNumber)
print("%s is still off. Unable to turn it back on!" %
device.name)
def checkAlwaysOn(self):
for device in self.devices:
self.alwaysOnDevice(device)
for device in self.alwaysOnDevices:
if not any(x.name == device for x in self.devices):
try:
print("OH MY GOD %s IS MISSING!" % device)
self.sendSMSMessage(
"%s not found in device list. Re-running discovery." %
device.name, self.notificationNumber)
self.discovery()
except AttributeError as err:
print(
"Weird error when trying to check always on. Device likely isn't a device?"
)
print(err)
self.discovery()
def update(self):
while self.bgRun:
self.reDiscover()
self.collectDeviceInfo()
self.checkAlwaysOn()
time.sleep(30)
action='store_false',
dest='execute_watch')
parser.set_defaults(endpoint='localhost',
port="9200",
protocol="http",
test_file='data.json',
user='******',
password='******')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
es = Elasticsearch(
[args.protocol + "://" + args.endpoint + ":" + args.port],
http_auth=(args.user, args.password),
ca_certs=args.cacert)
test = load_file(args.test_file)
if args.test_index:
# Load Mapping
try:
es.indices.delete(test['index'])
except Exception as err:
print("Unable to delete current dataset")
pass
index_template = load_file(test['mapping_file'])
for unneeded_keys in ['order', 'version', 'index_patterns']:
index_template.pop(unneeded_keys, None)
es.indices.create(index=test["index"], body=index_template)
def test_handler():
"""
Test the service handler
"""
es = Elasticsearch()
es.indices.delete('member', ignore_unavailable=True)
service.handler(
{
"Records": [{
"messageId":
"059f36b4-87a3-44ab-83d2-661975830a7d",
"body":
json.dumps({
"_id": "abc",
"firstName": "John",
"lastName": "Doe",
"hashedEmail": "0bc83cb571cd1c50ba6f3e8a78ef1346",
"email": "*****@*****.**"
})
}, {
"messageId":
"2e1424d4-f796-459a-8184-9c92662be6da",
"body":
json.dumps({
"_id":
"def",
"firstName":
"Jane",
"lastName":
"River",
"isPublic":
True,
"interests": ["Volley Ball", "Natation"],
"virtualStudies": [{
"_id": "1",
"name": "Virtual Study 1"
}, {
"_id": "12",
"name": "Virtual Study 2"
}]
}),
}]
}, {})
a = es.get('member', 'abc')
assert a.get('found')
assert a.get('_id') == 'abc'
assert a['_source'].get('firstName') == 'John'
assert a['_source'].get('lastName') == 'Doe'
assert not a['_source'].get('isPublic')
assert a['_source'].get(
'hashedEmail') == '0bc83cb571cd1c50ba6f3e8a78ef1346'
assert a['_source'].get('email') == '*****@*****.**'
b = es.get('member', 'def')
assert b.get('found')
assert b.get('_id') == 'def'
assert b['_source'].get('firstName') == 'Jane'
assert b['_source'].get('lastName') == 'River'
assert b['_source'].get('isPublic')
assert b['_source'].get('interests') == ["Volley Ball", "Natation"]
assert b['_source'].get('virtualStudies') == [{
"_id": "1",
"name": "Virtual Study 1"
}, {
"_id": "12",
"name": "Virtual Study 2"
}]
assert b['_source'].get('searchableInterests') == [{
"name": "Volley Ball"
}, {
"name": "Natation"
}]
# coding:utf-8
import json
from elasticsearch7 import Elasticsearch
# /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/
from MooseSpider.items import ShiCiItem
ES_URL = ['127.0.0.1:9200']
es = Elasticsearch(ES_URL)
class Article:
name = ""
author = ""
def test_ping():
pong = es.ping()
print(pong)
def test_delete(index):
if not es.indices.exists(index):
return False
else:
res = es.indices.delete(index=index)
return res['acknowledged']
#!/usr/bin/python3
from elasticsearch7 import Elasticsearch
import json
import sys
from constants import index, host, port
es = Elasticsearch([{'host': host, 'port': port}])
doc = {'size': 10000, 'query': {'match_all': {}}}
res = es.search(index=index, body=doc, scroll='1m')
number_of_results = res['hits']['total']['value']
print(f"number_of_results {number_of_results}")
hits = res['hits']['hits']
for hit in hits:
print(json.dumps(hit['_source'], indent=4, sort_keys=True))
def all():
es = Elasticsearch([{'host': host, 'port': port}])
es.indices.refresh(index)
res = es.cat.count(index, params={"format": "json"})
count = res[0]["count"]
return form.format(count=count)
from functions.setup_elastic_search import reload_es
print('Starting up...')
reboot = "LuaUPnP boot"
lastLine = ""
appendedLine = ""
vera_host = os.environ.get('VERA_HOST')
es_host = os.environ['ES_HOST']
es_port = os.environ['ES_PORT']
vera_log_index = os.environ['VERA_LOG_INDEX']
sleep_time = int(os.environ['SLEEP_TIME'])
skip_reload = os.environ['SKIP_RELOAD'] == "true"
startup = True
es = Elasticsearch(hosts=[{'host': es_host, 'port': es_port}])
url = "http://{}/cgi-bin/cmh/log.sh?Device=LuaUPnP".format(vera_host)
es_base_url = "http://{}:{}".format(es_host, es_port)
tag_line_exp = re.compile(
r'\d{2}[\t]\d{2}[-/]\d{2}[-/]\d{2} \d{1,2}:\d{2}:\d{2}.\d{3}')
date_exp = re.compile(r'\d{2}[-/]\d{2}[-/]\d{2} \d{1,2}:\d{2}:\d{2}.\d{3}')
index_tag = re.compile(r'^<.{10}>$')
if not skip_reload:
reload_es(es_base_url)
print("Elastic Search index {} has been deleted and reset...".format(vera_log_index))
def compose(message, timestamp, log_level):
doc = {
'timestamp': convert_to_utc(timestamp),
'message': message,
"""
Python 操作ES7 实现增删改查功能
"""
# elasticsearch集群服务器的地址
ES = [
'39.98.152.206:9200',
'39.98.152.206:9201'
]
# 创建elasticsearch客户端
es = Elasticsearch(
ES,
# 启动前嗅探es集群服务器
sniff_on_start=True,
# es集群服务器结点连接异常时是否刷新es节点信息
sniff_on_connection_fail=True,
# 每60秒刷新节点信息
sniffer_timeout=60
)
## 查询
query = {
"query": {
"match_all": {}
}
}
## 返回的是字典
ret_query = es.search(index='item', body=query)
print(ret_query['hits'])
import re
from elasticsearch7 import Elasticsearch
from flask import Flask, render_template, request
from pymongo import MongoClient
import config
elastic = Elasticsearch(config.ELASTIC_URL)
# Process elasticsearch hits and return flights records
def process_search(results):
records = []
total = 0
if results["hits"] and results["hits"]["hits"]:
total = results["hits"]["total"]
hits = results["hits"]["hits"]
for hit in hits:
record = hit["_source"]
records.append(record)
return records, total
# Calculate offsets for fetching lists of flights from MongoDB
def get_navigation_offsets(offset1, offset2, increment):
offsets = {}
offsets["Previous"] = {
"top_offset": max(offset2 - increment, 0),
"bottom_offset": max(offset1 - increment, 0),
#!/usr/bin/env python
from elasticsearch7 import Elasticsearch
import random
import time
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
index_name = "my-index"
day = 1
while True:
elem = {"day": day, "speed_of_load": random.uniform(0, 1) * 3}
res = es.index(index=index_name, body=elem)
assert res['_shards']['successful'] == 1
day += 1
print("added another record...")
time.sleep(1)
print(f"inserted {len(data_elements)} records")
)
parser.add_argument('--user', help='user')
parser.add_argument('--password', help='password')
parser.add_argument('--endpoint', help='endpoint')
parser.add_argument('--port', help='port')
parser.add_argument('--protocol', help='protocol')
parser.add_argument('--test_file', help='test file')
parser.set_defaults(endpoint='localhost',
port="9200",
protocol="http",
test_file='data.json',
user='******',
password='******')
args = parser.parse_args()
es = Elasticsearch([args.protocol + "://" + args.endpoint + ":" + args.port],
http_auth=(args.user, args.password))
def find_item(list, key):
for item in list:
if key in item:
return item
return None
with open(args.test_file, 'r') as test_file:
test = json.loads(test_file.read())
try:
es.indices.delete(test['index'])
except:
print("Unable to delete current dataset")
args = vars(get_args())
if not args['comparecsvs']:
if not args['index'] or not args['rootdir']:
print(
'--eshost1, --index and --rootdir cli args required (unless using --comparecsvs), use -h for help'
)
sys.exit(1)
# set up elasticsearch connections
es = Elasticsearch(hosts=args['eshost1'],
port=args['esport1'],
http_auth=(args['esuser1'], args['espass1']),
connection_class=Urllib3HttpConnection,
timeout=config['es_timeout'],
maxsize=config['es_maxsize'],
max_retries=config['es_max_retries'],
retry_on_timeout=True)
if args['eshost2']:
es2 = Elasticsearch(hosts=args['eshost2'],
port=args['esport2'],
http_auth=(args['esuser2'], args['espass2']),
connection_class=Urllib3HttpConnection,
timeout=config['es_timeout'],
maxsize=config['es_maxsize'],
max_retries=config['es_max_retries'],
retry_on_timeout=True)
else:
es2 = es
