async def add_elasticsearch_connection(self):
elasticsearch_host = None
elasticsearch_port = None
try:
elasticsearch_host = os.getenv("ELASTICSEARCH_HOST",
"redis.dev.muchneededllc.com")
elasticsearch_port = str(os.getenv("ELASTICSEARCH_PORT", 9200))
except OSError as e:
logger.error(
"Couldn't get environmental variables for elasticearch. " +
str(e))
exit(1)
try:
if self._loop is not None:
address = ':'.join([elasticsearch_host, elasticsearch_port])
self._elasticsearch_client = Elasticsearch([address],
loop=self._loop)
logger.info(self._elasticsearch_client)
self.elasticsearch_conn = True
logger.debug("Created Elasticsearch Client.")
else:
logger.error(
"Couldn't create elasticsearch client because loop hasn't been set."
)
except Exception as e:
logger.error("couldn't open elasticsearch.")
raise Exception(e)
async def related_word_extractor(parent_docid, doc_datetime, term, debug=False):
es = Elasticsearch(['%s:%d'%(es_ip, es_port)])
#print("%s %d" % ((es_ip, es_port)))
highlight_req = {
"_source" : [""],
"query": {
"bool": {
"filter": [
{
"term": {
"_id": parent_docid
}
},
{
"query_string": {
"query": term,
"fields": ["doc_title", "doc_content"],
"default_operator": "AND"
}
}
]
}
},
"highlight": {
"fields": {
"_all" : {},
"doc_title": {
"fragment_size": 30,
"number_of_fragments": 1,
"fragmenter": "simple"
},
"doc_content": {
"fragment_size": 30,
"number_of_fragments": 3,
"fragmenter": "simple"
}
}
}
}
result = await es.search(index=INDEX_DOCUMENTS+"-"+re.sub("-" , ".", doc_datetime[:doc_datetime.find("T")]), doc_type=TYPE_DOC, body=highlight_req)
related = []
if result['hits']['total']>0:
title_fragments = []
content_fragments = []
for a in result['hits']['hits']:
if 'doc_title' in a['highlight']:
title_fragments = [ fragment for fragment in a['highlight']['doc_title'] ]
if 'doc_content' in a['highlight']:
content_fragments = [ fragment for fragment in a['highlight']['doc_content'] ]
for f in (title_fragments+content_fragments):
related += await get_close_word(f, debug)
es.close()
return list(filter(lambda x:len(x)>1, list(sorted(set(related), key=lambda x:related.index(x)))))
def test__repr__(loop):
cl = Elasticsearch([], loop=loop)
assert repr(cl) == '<Elasticsearch [<Transport []>]>'
cl = Elasticsearch(['localhost:9200'], loop=loop)
assert repr(cl) == (
"<Elasticsearch [<Transport ["
"TCPEndpoint(scheme='http', host='localhost', port=9200)"
"]>]>")
def client(es_params, index, loop):
client = Elasticsearch([{'host': es_params['host']}], loop=loop)
try:
loop.run_until_complete(client.delete(index, '', ''))
except NotFoundError:
pass
yield client
client.close()
def setUp(self):
self._index = 'test_elasticsearch'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(self.cl.delete(self._index, '', ''))
except NotFoundError:
pass
async def analyze(text, debug=False):
es = Elasticsearch(['%s:%d' % (es_ip, es_port)])
client = IndicesClient(es)
index_name = await get_recent_index(INDEX_DOCUMENTS)
result = await client.analyze(index=index_name,
analyzer="korean",
body={"text": text})
ret = []
for x in result['tokens']:
token = x["token"]
type = x["type"]
# 명사 실질형태소와 외국어만 추출
if type not in [
"COMPOUND", "EOJEOL", "INFLECT", "VV", "VA", "VX", "VCP",
"VCN", "NNB", "E", "JKS", "JKC", "JKG", "JKO", "JKB", "JKV",
"JKQ", "JX", "JC", "EP", "EF", "EC", "ETN", "ETM", "XPN",
"XSN", "XSV", "XSA", "SF", "SE", "SS", "SN", "SP", "SO", "SW",
"SH"
]:
if debug: print("{}==>{}/{}".format(text, token, type))
if (type in ["VV", "VA"]):
ret.append(token[:token.find("/V")] + "다")
else:
ret.append(token)
else:
if debug: print("XXX {}==>{}/{}".format(text, token, type))
return "".join(ret)
def setUp(self):
self._index = 'elastic_search'
self.repo_name = 'test_repo'
self.repo_path = self._create_temp_dir()
# otherwise elasticsearch can't access it
os.chmod(self.repo_path, 0o777)
self.snapshot_name = 'test_snapshot'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def client(es_params, loop):
with closing(Elasticsearch([{'host': es_params['host']}], loop=loop)) as c:
INDEX = 'test_elasticsearch'
try:
loop.run_until_complete(c.delete(INDEX, '', ''))
except NotFoundError:
pass
yield c
def setUp(self):
self._index = "elastic_search"
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{"host": "localhost"}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def client(es_params, loop, repo_name, snapshot_name):
client = Elasticsearch([{'host': es_params['host']}], loop=loop)
try:
loop.run_until_complete(client.delete(INDEX, '', ''))
except NotFoundError:
pass
yield client
# cleaning up just in case
try:
loop.run_until_complete(
client.snapshot.delete(repo_name, snapshot_name))
except NotFoundError:
pass
try:
loop.run_until_complete(client.snapshot.delete_repository(repo_name))
except NotFoundError:
pass
client.close()
async def get_recent_index(index):
es = Elasticsearch(['%s:%d' % (es_ip, es_port)])
cat2es = CatClient(es)
result = await cat2es.indices(index, h="index")
'''
es_conn = http.client.HTTPConnection(es_ip, es_port)
es_conn.request("GET", "_cat/indices/"+index+"?h=index&s=index:desc", "", { "Content-Type" : "application/json" })
es_result = es_conn.getresponse().read().decode('utf-8')
'''
es_result = result
idx_list = sorted([idx for idx in es_result.split("\n")], reverse=True)
if len(es_result) > 0:
return idx_list[0].strip()
else:
raise EsError
def setUp(self):
self._index = 'elastic_search'
self.repo_name = 'test_repo'
self.repo_path = self._create_temp_dir()
# otherwise elasticsearch can't access it
os.chmod(self.repo_path, 0o777)
self.snapshot_name = 'test_snapshot'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
class TestNodes(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_info(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.info()
self.assertIn('cluster_name', ret)
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.stats()
self.assertIn('nodes', ret)
self.assertTrue(len(ret['nodes']) > 0)
self.loop.run_until_complete(go())
def test_hot_threads(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.hot_threads()
self.assertIn('cpu usage by thread', ret)
self.loop.run_until_complete(go())
async def update_emotions(index_name, emotion_id, pos_neg):
logger.debug("[update] %s, %s" % (emotion_id, pos_neg))
request_body = {
"script": {
"inline":
"ctx._source.put('emotion_type', params.emotion_type); ctx._source.put('upd_datetime', params.upd_datetime);",
"lang": "painless",
"params": {
"emotion_type": pos_neg,
"upd_datetime": str(dt.now().strftime('%Y-%m-%dT%H:%M:%S'))
}
}
}
es = Elasticsearch(['%s:%d' % (es_ip, es_port)])
r = await es.update(index=index_name,
doc_type=TYPE_DOC,
id=emotion_id,
body=request_body,
refresh=True)
class TestNodes(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_info(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.info()
self.assertIn('cluster_name', ret)
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.stats()
self.assertIn('nodes', ret)
self.assertTrue(len(ret['nodes']) > 0)
self.loop.run_until_complete(go())
def test_hot_threads(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.nodes.hot_threads()
self.assertIn('cpu usage by thread', ret)
self.loop.run_until_complete(go())
async def find_to_which_index(topic_id):
whichIndex = None
request_body = {"query": {"term": {"_id": topic_id}}}
es = Elasticsearch(['%s:%d' % (es_ip, es_port)])
es_result = await es.search(TOPICS_TO_SEARCH,
doc_type="doc",
body=request_body)
'''
es_conn = http.client.HTTPConnection(es_ip, es_port)
es_conn.request("POST", "/"+INDEX_DOCUMENTS+"/"+TYPE_DOC+"/_search", json.dumps(request_body), { "Content-Type" : "application/json" })
json_result = json.loads(es_conn.getresponse().read())
'''
#json_result = json.loads(es_result)
if 'hits' in es_result:
if es_result['hits']['total'] > 0:
whichIndex = es_result['hits']['hits'][0]['_index']
else:
#print('No hits.')
whichIndex = await get_recent_index(TOPICS_TO_SEARCH)
else:
raise EsError
return whichIndex
class TestSnapshot(unittest.TestCase):
def _create_temp_dir(self):
'''
Sice elasticsearch may be launched under the other user,
tempfile.TemporaryDirectory can't be used, because python
can't cleanup it after elasticsearch user.
So we are just leaving it there.
'''
characters = "abcdefghijklmnopqrstuvwxyz0123456789_"
temp_dir = tempfile.gettempdir()
temp_prefix = '/' + tempfile.gettempprefix()
temp_name = temp_prefix + ''.join(
[random.choice(characters) for _ in range(8)]
)
dir_path = os.path.join(temp_dir + temp_name)
os.makedirs(dir_path)
return dir_path
def _cleanup_dirs(self, dir_path):
try:
shutil.rmtree(dir_path)
except PermissionError:
# if subdirs were created by other user
pass
def setUp(self):
self._index = 'elastic_search'
self.repo_name = 'test_repo'
self.repo_path = self._create_temp_dir()
# otherwise elasticsearch can't access it
os.chmod(self.repo_path, 0o777)
self.snapshot_name = 'test_snapshot'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self._cleanup_dirs(self.repo_path)
# cleaning up just in case
try:
self.loop.run_until_complete(
self.cl.snapshot.delete(self.repo_name, self.snapshot_name))
except NotFoundError:
pass
try:
self.loop.run_until_complete(
self.cl.snapshot.delete_repository(self.repo_name))
except NotFoundError:
pass
# close loop
self.loop.close()
def test_repository(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.snapshot.get_repository()
self.assertFalse(ret)
b = {
"type": "fs",
"settings": {
"location": self.repo_path,
}
}
ret = yield from self.cl.snapshot.create_repository(
self.repo_name, b
)
self.assertTrue(ret['acknowledged'])
ret = yield from self.cl.snapshot.get_repository(
self.repo_name
)
self.assertEqual(ret[self.repo_name], b)
ret = yield from self.cl.snapshot.delete_repository(
self.repo_name
)
self.assertTrue(ret['acknowledged'])
self.loop.run_until_complete(go())
def test_snapshot(self):
@asyncio.coroutine
def go():
# creating index
yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
},
'1'
)
# creating repo
repo_body = {
"type": "fs",
"settings": {
"location": self.repo_path,
}
}
ret = yield from self.cl.snapshot.create_repository(
self.repo_name, repo_body
)
self.assertTrue(ret['acknowledged'])
# creating snapshot
yield from self.cl.snapshot.create(
self.repo_name, self.snapshot_name,
wait_for_completion=True
)
# checking that snapshot was created
ret = yield from self.cl.snapshot.get(
self.repo_name, self.snapshot_name,
)
self.assertEqual(
ret['snapshots'][0]['snapshot'],
self.snapshot_name
)
self.assertEqual(
ret['snapshots'][0]['state'],
'SUCCESS'
)
# restoring snapshot
restore_body = {
"indices": self._index,
}
yield from self.cl.indices.close(self._index)
ret = yield from self.cl.snapshot.restore(
self.repo_name, self.snapshot_name,
body=restore_body
)
self.assertTrue(ret['accepted'])
# deleting snapshot
ret = yield from self.cl.snapshot.delete(
self.repo_name, self.snapshot_name,
)
self.assertTrue(ret['acknowledged'])
# deleting repo
ret = yield from self.cl.snapshot.delete_repository(
self.repo_name
)
self.assertTrue(ret['acknowledged'])
self.loop.run_until_complete(go())
def test_status(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.snapshot.status()
self.assertEqual({'snapshots': []}, ret)
self.loop.run_until_complete(go())
from aioes import Elasticsearch from .settings import ELASTICSEARCH_ENDPOINTS elastic = Elasticsearch(ELASTICSEARCH_ENDPOINTS)
def conn(self):
if self._conn is None:
self._conn = Elasticsearch(**self.settings['connection_settings'])
return self._conn
class TestIndices(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_analize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.analyze(
text='this, is a test 125 !',
analyzer='standard',
filters='lowercase')
self.assertEqual(len(data['tokens']), 5)
self.assertEqual(data['tokens'][0]['token'], 'this')
self.assertEqual(data['tokens'][1]['token'], 'is')
self.assertEqual(data['tokens'][2]['token'], 'a')
self.assertEqual(data['tokens'][3]['token'], 'test')
self.assertEqual(data['tokens'][4]['token'], '125')
self.assertEqual(data['tokens'][4]['type'], '<NUM>')
data = yield from self.cl.indices.analyze(
text='this is a <b>test</b>',
tokenizer='keyword',
token_filters='lowercase',
char_filters='html_strip',
prefer_local=True)
self.assertEqual(data['tokens'][0]['token'], 'this is a test')
with self.assertRaises(RequestError):
yield from self.cl.indices.analyze(
analyzer='standard',
filters='lowercase',
field='w')
self.loop.run_until_complete(go())
def test_create(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.create(
self._index, timeout=1000, master_timeout=1000)
self.assertTrue(data['acknowledged'])
self.loop.run_until_complete(go())
def test_refresh(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.refresh(self._index)
self.assertIn('_shards', data, data)
yield from self.cl.indices.refresh(
self._index,
allow_no_indices=False, expand_wildcards='closed',
ignore_unavailable=True, ignore_indices='', force=True)
with self.assertRaises(TypeError):
yield from self.cl.indices.refresh(
self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.refresh(
self._index, expand_wildcards='1')
self.loop.run_until_complete(go())
def test_flush(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.flush(self._index)
self.assertIn('_shards', data, data)
yield from self.cl.indices.flush(
self._index, full=True,
allow_no_indices=False, expand_wildcards='closed',
ignore_unavailable=True, ignore_indices='', force=True)
with self.assertRaises(TypeError):
yield from self.cl.indices.flush(
self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.flush(
self._index, expand_wildcards='1')
self.loop.run_until_complete(go())
def test_open(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
data = yield from self.cl.indices.open(
self._index, timeout=1000, master_timeout=1000,
allow_no_indices=False, expand_wildcards='closed',
ignore_unavailable=True)
self.assertTrue(data['acknowledged'], data)
data = yield from self.cl.indices.open(self._index)
with self.assertRaises(TypeError):
yield from self.cl.indices.open(self._index,
expand_wildcards=1,
ignore_unavailable=True)
with self.assertRaises(ValueError):
yield from self.cl.indices.open(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_close(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type',
MESSAGE,
'1')
yield from self.cl.cluster.health(
self._index,
wait_for_status='yellow')
data = yield from self.cl.indices.close(self._index)
self.assertTrue(data['acknowledged'], data)
data = yield from self.cl.indices.close(
self._index,
timeout='1s', master_timeout='1s',
expand_wildcards='open',
allow_no_indices=True,
ignore_unavailable=True)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.close(
self._index,
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.close(
self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_delete(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.delete(self._index)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(NotFoundError):
yield from self.cl.indices.delete(
self._index, timeout='1s', master_timeout='1s')
self.assertTrue(data['acknowledged'], data)
self.loop.run_until_complete(go())
def test_exists(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.exists(
self._index,
allow_no_indices=False,
expand_wildcards='closed',
ignore_unavailable=False,
local=False)
self.assertTrue(data)
data = yield from self.cl.indices.exists(self._index+'123')
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists(
self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists(
self._index, expand_wildcards='1')
self.loop.run_until_complete(go())
def test_exists_type(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.exists_type(
self._index, 'type', allow_no_indices=False)
self.assertTrue(data)
data = yield from self.cl.indices.exists_type(
self._index, 'ert', expand_wildcards='open')
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists_type(
self._index, '', expand_wildcards=1,
allow_no_indices=True,
ignore_unavailable=True,
ignore_indices=True,
local=True)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists_type(
self._index, '', expand_wildcards='1')
self.loop.run_until_complete(go())
def test_get_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.get_settings()
self.assertIn(self._index, data)
data = yield from self.cl.indices.get_settings(
expand_wildcards='open',
ignore_indices='',
flat_settings=False,
ignore_unavailable=False,
local=True)
self.assertIn(self._index, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.get_settings(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.get_settings(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_put_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.put_settings(
{"index": {"number_of_replicas": 2}}, self._index)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(RequestError):
yield from self.cl.indices.put_settings(
{"index": {"number_of_replicas": 2}},
allow_no_indices=True,
expand_wildcards='open',
flat_settings=False,
ignore_unavailable=False,
master_timeout='1s')
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.put_settings(
{}, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.put_settings(
{}, expand_wildcards='1')
self.loop.run_until_complete(go())
def test_status(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.status()
self.assertIn('indices', data)
data = yield from self.cl.indices.status(
ignore_indices='',
allow_no_indices=True,
recovery=False,
snapshot=False,
operation_threading='',
expand_wildcards='open',
ignore_unavailable=False,
human=True)
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.status(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.status(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.stats()
self.assertIn('indices', data, data)
data = yield from self.cl.indices.stats(
metric='_all',
completion_fields='*',
docs=1,
fielddata_fields='*',
fields='*',
groups='*',
allow_no_indices=True,
expand_wildcards='open',
ignore_indices=False,
ignore_unavailable=True,
level='cluster',
types='*',
human=True)
self.assertIn('_all', data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(expand_wildcards='1')
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(level=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(level='1')
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(metric=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(metric='1')
self.loop.run_until_complete(go())
def test_segments(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.segments()
self.assertIn('indices', data, data)
self.assertIn('_shards', data, data)
data = yield from self.cl.indices.segments(
allow_no_indices=True,
ignore_indices=True,
ignore_unavailable=True,
expand_wildcards='open',
human=True)
self.assertIn('indices', data, data)
self.assertIn('_shards', data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.segments(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.segments(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_optimize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.optimize()
self.assertIn('_shards', data)
data = yield from self.cl.indices.optimize(
allow_no_indices=True,
expand_wildcards='open',
ignore_indices=True,
ignore_unavailable=True,
max_num_segments=0,
only_expunge_deletes=True,
operation_threading='',
wait_for_merge=False,
force=True,
flush=True)
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.optimize(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.optimize(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_validate_query(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.validate_query()
self.assertIn('_shards', data)
yield from self.cl.indices.validate_query(
explain=True,
allow_no_indices=True,
q='',
ignore_indices=True,
source='',
operation_threading='',
expand_wildcards='open',
ignore_unavailable=False)
with self.assertRaises(TypeError):
yield from self.cl.indices.validate_query(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.validate_query(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_clear_cache(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.clear_cache()
self.assertIn('_shards', data)
yield from self.cl.indices.clear_cache(
field_data=True,
fielddata=True,
recycler=True,
id_cache=True,
filter_keys='',
filter_cache=True,
filter=False,
fields='',
id=False,
allow_no_indices=False,
ignore_indices=False,
ignore_unavailable=True,
expand_wildcards='open')
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.clear_cache(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.clear_cache(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_recovery(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.recovery()
self.assertIn(self._index, data)
data = yield from self.cl.indices.recovery(
active_only=False,
detailed=True,
human=True)
self.loop.run_until_complete(go())
def test_mapping(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
mapping = {
'testdoc': {
'properties': {
'message': {
'type': 'string',
}
}
}
}
# PUT
data = yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
self.assertTrue(data['acknowledged'])
# GET
data = yield from self.cl.indices.get_mapping(
self._index,
'testdoc',
)
self.assertEqual(data['elastic_search']['mappings'], mapping)
# DELETE
yield from self.cl.indices.delete_mapping(
self._index,
'testdoc',
)
data = yield from self.cl.indices.get_mapping(
self._index,
'testdoc',
)
self.assertFalse(data)
self.loop.run_until_complete(go())
def test_get_field_mapping(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
rt = yield from self.cl.indices.get_field_mapping(
'message', index=self._index
)
self.assertEqual(
# dude, you are so deep
rt[self._index]['mappings']['type']['message']['mapping'],
{'message': {'type': 'string'}}
)
self.loop.run_until_complete(go())
def test_warmers(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertFalse(a)
b = {
"query": {
"match_all": {}
},
"aggs": {
"aggs_1": {
"terms": {
"field": "message"
}
}
}
}
yield from self.cl.indices.put_warmer(
index=self._index, name='warmer', body=b
)
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertIn('warmer', a[self._index]['warmers'].keys())
yield from self.cl.indices.delete_warmer(
name='warmer', index=self._index
)
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertFalse(a)
self.loop.run_until_complete(go())
def test_aliases(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
al = yield from self.cl.indices.exists_alias('alias')
self.assertFalse(al)
al = yield from self.cl.indices.get_alias('alias')
self.assertEqual({}, al)
al = yield from self.cl.indices.get_aliases('alias')
self.assertEqual({}, al)
yield from self.cl.indices.put_alias('alias', self._index)
al = yield from self.cl.indices.exists_alias('alias')
self.assertTrue(al)
yield from self.cl.indices.update_aliases(body={
"actions": [
{"remove": {"index": self._index, "alias": "alias"}},
{"add": {"index": self._index, "alias": "alias2"}}
]
})
al = yield from self.cl.indices.exists_alias('alias2')
self.assertTrue(al)
yield from self.cl.indices.delete_alias(self._index, 'alias2')
al = yield from self.cl.indices.get_aliases('alias')
self.assertFalse(al)
self.loop.run_until_complete(go())
def test_templates(self):
@asyncio.coroutine
def go():
b = {
"template": self._index,
"settings": {
"number_of_shards": '1'
},
}
t = yield from self.cl.indices.exists_template('template')
self.assertFalse(t)
yield from self.cl.indices.put_template('template', b)
t = yield from self.cl.indices.exists_template('template')
self.assertTrue(t)
t = yield from self.cl.indices.get_template('template')
self.assertEqual(
t['template']['settings']['index.number_of_shards'],
b['settings']['number_of_shards']
)
yield from self.cl.indices.delete_template('template')
t = yield from self.cl.indices.exists_template('template')
self.assertFalse(t)
self.loop.run_until_complete(go())
class TestCluster(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_health(self):
@asyncio.coroutine
def go():
data = yield from self.cl.cluster.health()
self.assertIn('status', data)
data = yield from self.cl.cluster.health(
level='indices',
local=True,
master_timeout='1s',
timeout='1s',
wait_for_active_shards=1,
wait_for_nodes='>2',
wait_for_relocating_shards=0)
self.assertIn('status', data)
with self.assertRaises(TypeError):
yield from self.cl.cluster.health(level=1)
with self.assertRaises(ValueError):
yield from self.cl.cluster.health(level='1')
with self.assertRaises(TypeError):
yield from self.cl.cluster.health(wait_for_status=1)
with self.assertRaises(ValueError):
yield from self.cl.cluster.health(wait_for_status='1')
self.loop.run_until_complete(go())
def test_pending_tasks(self):
@asyncio.coroutine
def go():
data = yield from self.cl.cluster.pending_tasks()
self.assertIn('tasks', data)
self.loop.run_until_complete(go())
def test_state(self):
@asyncio.coroutine
def go():
data = yield from self.cl.cluster.state()
self.assertIn('routing_nodes', data)
self.assertIn('master_node', data)
# create index
yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
},
'1'
)
data = yield from self.cl.cluster.state(index=self._index)
self.assertIn('routing_nodes', data)
self.assertIn('master_node', data)
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
data = yield from self.cl.cluster.stats()
self.assertIn('indices', data)
self.assertIn('nodes', data)
self.loop.run_until_complete(go())
def test_reroute(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
},
'1'
)
# get node's name
nodes = yield from self.cl.nodes.info()
node = list(nodes['nodes'].keys())[0]
b = {
"commands": [
{
"cancel": {
"index": self._index,
"shard": 1,
"node": node,
'allow_primary': True,
}
}
]
}
data = yield from self.cl.cluster.reroute(body=b, dry_run=True)
self.assertTrue(data['acknowledged'])
self.loop.run_until_complete(go())
def test_get_settings(self):
@asyncio.coroutine
def go():
data = yield from self.cl.cluster.get_settings()
self.assertIn('persistent', data)
self.assertIn('transient', data)
self.loop.run_until_complete(go())
def test_put_settings(self):
@asyncio.coroutine
def go():
b = {
"transient": {
"cluster.routing.allocation.enable": "all"
}
}
data = yield from self.cl.cluster.put_settings(b)
routing_settings = data['transient']['cluster']['routing']
self.assertEqual(
routing_settings['allocation']['enable'],
'all'
)
self.loop.run_until_complete(go())
def insert_topics(data):
some_bulks = ''
bulk_result = 0
try:
result = teaclient.request(data)
#print(result)
root = et.fromstring(result)
status = root.findall(
"./results/result[@name='status']")[0].text if len(
root.findall("./results/result[@name='status']")) > 0 else ''
#print(">>> Tea client response : %s" % status)
if status == "success" and len(
root.findall("./results/result[@name='keywords']")) > 0:
result_scd = root.findall(
"./results/result[@name='keywords']")[0].text
terms = ""
verbs = ""
for line in result_scd.split("\n"):
if line.startswith("<TERMS>"):
terms = line.replace("<TERMS>", "") # 하늘:387^테스트:14^도움:11
elif line.startswith("<VERBS>"):
verbs = line.replace("<VERBS>", "") # 하늘:387^테스트:14^도움:11
#print("### terms : %s" % terms)
# <TERMS>
#t = asyncio.ensure_future(time_log())
terms = [('NN', term)
for term in terms.split(teaclient.ITEM_DELIMITER)]
verbs = [('VV', verb)
for verb in verbs.split(teaclient.ITEM_DELIMITER)]
from elasticsearch import Elasticsearch
es_client = Elasticsearch(":".join([es_ip, str(es_port)]))
try:
fts = [make_bulk(t, data) for t in (terms + verbs)]
#t.cancel()
some_bulks = yield from asyncio.gather(*fts)
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0, some_bulks)),
refresh=True)[0]
except EsError as e:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" % (str(e), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
break
except EsError as e:
logger.error("[insert_topics] %s (retry:%d)" %
(str(e), retry))
continue
except exceptions.ConnectionTimeout as timeoutError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(timeoutError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
break
except exceptions.ConnectionTimeout as timeoutError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(timeoutError), retry))
continue
except aiohttp.client_exceptions.ClientConnectorError as connectError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
break
except aiohttp.client_exceptions.ClientConnectorError as connectError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectError), retry))
continue
except OSError as oserror:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(oserror), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
break
except OSError as oserror:
logger.error("[insert_topics] %s (retry:%d)" %
(str(oserror), retry))
continue
except urllib3.exceptions.NewConnectionError as connectionError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectionError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
break
except urllib3.exceptions.NewConnectionError as connectionError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectionError), retry))
continue
except:
ex = traceback.format_exc()
logger.error(
"[insert_topics] unknown error. Traceback >> %s " % ex)
logger.debug("%d are successfully inserted." % bulk_result)
except ParseError as xmlerror:
logger.error("[insert_topics] TeaClient failed. (%s)" % str(xmlerror))
logger.error("==============> teaclient's xml response : %s" % result)
class TestSnapshot(unittest.TestCase):
def _create_temp_dir(self):
'''
Sice elasticsearch may be launched under the other user,
tempfile.TemporaryDirectory can't be used, because python
can't cleanup it after elasticsearch user.
So we are just leaving it there.
'''
characters = "abcdefghijklmnopqrstuvwxyz0123456789_"
temp_dir = tempfile.gettempdir()
temp_prefix = '/' + tempfile.gettempprefix()
temp_name = temp_prefix + ''.join(
[random.choice(characters) for _ in range(8)])
dir_path = os.path.join(temp_dir + temp_name)
os.makedirs(dir_path)
return dir_path
def _cleanup_dirs(self, dir_path):
try:
shutil.rmtree(dir_path)
except PermissionError:
# if subdirs were created by other user
pass
def setUp(self):
self._index = 'elastic_search'
self.repo_name = 'test_repo'
self.repo_path = self._create_temp_dir()
# otherwise elasticsearch can't access it
os.chmod(self.repo_path, 0o777)
self.snapshot_name = 'test_snapshot'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self._cleanup_dirs(self.repo_path)
# cleaning up just in case
try:
self.loop.run_until_complete(
self.cl.snapshot.delete(self.repo_name, self.snapshot_name))
except NotFoundError:
pass
try:
self.loop.run_until_complete(
self.cl.snapshot.delete_repository(self.repo_name))
except NotFoundError:
pass
# close loop
self.loop.close()
def test_repository(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.snapshot.get_repository()
self.assertFalse(ret)
b = {
"type": "fs",
"settings": {
"location": self.repo_path,
}
}
ret = yield from self.cl.snapshot.create_repository(
self.repo_name, b)
self.assertTrue(ret['acknowledged'])
ret = yield from self.cl.snapshot.get_repository(self.repo_name)
self.assertEqual(ret[self.repo_name], b)
ret = yield from self.cl.snapshot.delete_repository(self.repo_name)
self.assertTrue(ret['acknowledged'])
self.loop.run_until_complete(go())
def test_snapshot(self):
@asyncio.coroutine
def go():
# creating index
yield from self.cl.create(self._index, 'tweet', {
'user': '******',
}, '1')
# creating repo
repo_body = {
"type": "fs",
"settings": {
"location": self.repo_path,
}
}
ret = yield from self.cl.snapshot.create_repository(
self.repo_name, repo_body)
self.assertTrue(ret['acknowledged'])
# creating snapshot
yield from self.cl.snapshot.create(self.repo_name,
self.snapshot_name,
wait_for_completion=True)
# checking that snapshot was created
ret = yield from self.cl.snapshot.get(
self.repo_name,
self.snapshot_name,
)
self.assertEqual(ret['snapshots'][0]['snapshot'],
self.snapshot_name)
self.assertEqual(ret['snapshots'][0]['state'], 'SUCCESS')
# restoring snapshot
restore_body = {
"indices": self._index,
}
yield from self.cl.indices.close(self._index)
ret = yield from self.cl.snapshot.restore(self.repo_name,
self.snapshot_name,
body=restore_body)
self.assertTrue(ret['accepted'])
# deleting snapshot
ret = yield from self.cl.snapshot.delete(
self.repo_name,
self.snapshot_name,
)
self.assertTrue(ret['acknowledged'])
# deleting repo
ret = yield from self.cl.snapshot.delete_repository(self.repo_name)
self.assertTrue(ret['acknowledged'])
self.loop.run_until_complete(go())
def test_status(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.snapshot.status()
self.assertEqual({'snapshots': []}, ret)
self.loop.run_until_complete(go())
def insertTopics(es_result):
if 'hits' in es_result:
######## topic_class가 VV인 항목들을 검색해서 그 결과를 넘겨주고 bulk를 만듦.
fts = [ makeUpdateBulks(x) for x in es_result['hits']['hits'] ]
some_bulks = yield from asyncio.gather(*fts)
######## BULK INSERT
from elasticsearch import Elasticsearch
es_client=Elasticsearch(":".join([es_ip, str(es_port)]))
bulk_result=0
try:
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
except EsError as e:
retry = 0
logger.error("[insertTopics] %s (retry:%d)"%(str(e), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
break
except EsError as e:
logger.error("[insertTopics] %s (retry:%d)"%(str(e), retry))
continue
except exceptions.ConnectionTimeout as timeoutError:
retry = 0
logger.error("[insertTopics] %s (retry:%d)"%(str(timeoutError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
break
except exceptions.ConnectionTimeout as timeoutError:
logger.error("[insertTopics] %s (retry:%d)"%(str(timeoutError), retry))
continue
except aiohttp.client_exceptions.ClientConnectorError as connectError:
retry = 0
logger.error("[insertTopics] %s (retry:%d)"%(str(connectError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
break
except aiohttp.client_exceptions.ClientConnectorError as connectError:
logger.error("[insertTopics] %s (retry:%d)"%(str(connectError), retry))
continue
except OSError as oserror:
retry = 0
logger.error("[insertTopics] %s (retry:%d)"%(str(oserror), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
break
except OSError as oserror:
logger.error("[insertTopics] %s (retry:%d)"%(str(oserror), retry))
continue
except urllib3.exceptions.NewConnectionError as connectionError:
retry = 0
logger.error("[insertTopics] %s (retry:%d)"%(str(connectionError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(es_client, list(filter(lambda x:x is not None, some_bulks)), refresh=True)[0]
break
except urllib3.exceptions.NewConnectionError as connectionError:
logger.error("[insertTopics] %s (retry:%d)"%(str(connectionError), retry))
continue
except:
ex = traceback.format_exc()
logger.error("[insertTopics] unknown error. Traceback >> %s " % ex)
logger.debug("%d are successfully inserted."%bulk_result)
class TestCat(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_aliases(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.aliases(v=True)
self.assertIn('alias', ret)
self.assertIn('index', ret)
self.loop.run_until_complete(go())
def test_allocation(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.allocation(v=True)
self.assertIn('disk.percent', ret)
self.loop.run_until_complete(go())
def test_count(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.count(v=True)
self.assertIn('timestamp', ret)
self.assertIn('count', ret)
# testing for index
yield from self.cl.create(self._index, 'tweet', {
'user': '******',
}, '1')
ret = yield from self.cl.cat.count(self._index, v=True)
self.assertIn('timestamp', ret)
self.assertIn('count', ret)
self.loop.run_until_complete(go())
def test_health(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.health(v=True)
self.assertIn('timestamp', ret)
self.assertIn('node.total', ret)
self.loop.run_until_complete(go())
def test_help(self):
pattern = textwrap.dedent("""\
=^.^=
/_cat/allocation
/_cat/shards
/_cat/shards/{index}
/_cat/master
/_cat/nodes
/_cat/indices
/_cat/indices/{index}
/_cat/segments
/_cat/segments/{index}
/_cat/count
/_cat/count/{index}
/_cat/recovery
/_cat/recovery/{index}
/_cat/health
/_cat/pending_tasks
/_cat/aliases
/_cat/aliases/{alias}
/_cat/thread_pool
/_cat/plugins
/_cat/fielddata
/_cat/fielddata/{fields}
""")
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.help(help=True)
self.assertEqual(pattern, ret)
self.loop.run_until_complete(go())
def test_indices(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.indices(v=True)
self.assertIn('health', ret)
self.assertIn('index', ret)
self.loop.run_until_complete(go())
def test_master(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.master(v=True)
self.assertIn('host', ret)
self.assertIn('ip', ret)
self.loop.run_until_complete(go())
def test_nodes(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.nodes(v=True)
self.assertIn('load', ret)
self.assertIn('name', ret)
self.loop.run_until_complete(go())
def test_recovery(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.recovery(v=True)
self.assertIn('index', ret)
self.assertIn('files', ret)
self.loop.run_until_complete(go())
def test_shards(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.shards(v=True)
self.assertIn('index', ret)
self.assertIn('node', ret)
self.loop.run_until_complete(go())
def test_segments(self):
@asyncio.coroutine
def go():
yield from self.cl.create(self._index, 'tweet', {
'user': '******',
}, '1')
ret = yield from self.cl.cat.segments(index=self._index, v=True)
self.assertIn('index', ret)
self.assertIn('segment', ret)
self.loop.run_until_complete(go())
def test_pending_tasks(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.pending_tasks(v=True)
self.assertIn('insertOrder', ret)
self.assertIn('priority', ret)
self.loop.run_until_complete(go())
def test_thread_pool(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.thread_pool(v=True)
self.assertIn('host', ret)
self.assertIn('ip', ret)
self.loop.run_until_complete(go())
def test_fielddata(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.fielddata(v=True)
self.assertIn('id', ret)
self.assertIn('total', ret)
self.loop.run_until_complete(go())
def test_plugins(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.plugins(v=True)
self.assertIn('name', ret)
self.assertIn('component', ret)
self.loop.run_until_complete(go())
class TestIndices(unittest.TestCase):
def setUp(self):
self._index = "elastic_search"
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{"host": "localhost"}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_analize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.analyze(
text="this, is a test 125 !", analyzer="standard", filters="lowercase"
)
self.assertEqual(len(data["tokens"]), 5)
self.assertEqual(data["tokens"][0]["token"], "this")
self.assertEqual(data["tokens"][1]["token"], "is")
self.assertEqual(data["tokens"][2]["token"], "a")
self.assertEqual(data["tokens"][3]["token"], "test")
self.assertEqual(data["tokens"][4]["token"], "125")
self.assertEqual(data["tokens"][4]["type"], "<NUM>")
data = yield from self.cl.indices.analyze(
text="this is a <b>test</b>",
tokenizer="keyword",
token_filters="lowercase",
char_filters="html_strip",
prefer_local=True,
)
self.assertEqual(data["tokens"][0]["token"], "this is a test")
with self.assertRaises(RequestError):
yield from self.cl.indices.analyze(analyzer="standard", filters="lowercase", field="w")
self.loop.run_until_complete(go())
def test_create(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.create(self._index, timeout=1000, master_timeout=1000)
self.assertTrue(data["acknowledged"])
self.loop.run_until_complete(go())
def test_refresh(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
data = yield from self.cl.indices.refresh(self._index)
self.assertIn("_shards", data, data)
yield from self.cl.indices.refresh(
self._index,
allow_no_indices=False,
expand_wildcards="closed",
ignore_unavailable=True,
ignore_indices="",
force=True,
)
with self.assertRaises(TypeError):
yield from self.cl.indices.refresh(self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.refresh(self._index, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_flush(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
data = yield from self.cl.indices.flush(self._index)
self.assertIn("_shards", data, data)
yield from self.cl.indices.flush(
self._index,
full=True,
allow_no_indices=False,
expand_wildcards="closed",
ignore_unavailable=True,
ignore_indices="",
force=True,
)
with self.assertRaises(TypeError):
yield from self.cl.indices.flush(self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.flush(self._index, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_open(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
data = yield from self.cl.indices.open(
self._index,
timeout=1000,
master_timeout=1000,
allow_no_indices=False,
expand_wildcards="closed",
ignore_unavailable=True,
)
self.assertTrue(data["acknowledged"], data)
data = yield from self.cl.indices.open(self._index)
with self.assertRaises(TypeError):
yield from self.cl.indices.open(self._index, expand_wildcards=1, ignore_unavailable=True)
with self.assertRaises(ValueError):
yield from self.cl.indices.open(self._index, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_close(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
yield from self.cl.cluster.health(self._index, wait_for_status="yellow")
data = yield from self.cl.indices.close(self._index)
self.assertTrue(data["acknowledged"], data)
data = yield from self.cl.indices.close(
self._index,
timeout="1s",
master_timeout="1s",
expand_wildcards="open",
allow_no_indices=True,
ignore_unavailable=True,
)
self.assertTrue(data["acknowledged"], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.close(self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.close(self._index, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_delete(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
data = yield from self.cl.indices.delete(self._index)
self.assertTrue(data["acknowledged"], data)
with self.assertRaises(NotFoundError):
yield from self.cl.indices.delete(self._index, timeout="1s", master_timeout="1s")
self.assertTrue(data["acknowledged"], data)
self.loop.run_until_complete(go())
def test_exists(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
data = yield from self.cl.indices.exists(
self._index, allow_no_indices=False, expand_wildcards="closed", ignore_unavailable=False, local=False
)
self.assertTrue(data)
data = yield from self.cl.indices.exists(self._index + "123")
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists(self._index, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists(self._index, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_exists_type(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
yield from self.cl.cluster.health(self._index)
data = yield from self.cl.indices.exists_type(self._index, "type", allow_no_indices=False)
self.assertTrue(data)
data = yield from self.cl.indices.exists_type(self._index, "ert", expand_wildcards="open")
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists_type(
self._index,
"",
expand_wildcards=1,
allow_no_indices=True,
ignore_unavailable=True,
ignore_indices=True,
local=True,
)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists_type(self._index, "", expand_wildcards="1")
self.loop.run_until_complete(go())
def test_get_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.get_settings()
self.assertIn(self._index, data)
data = yield from self.cl.indices.get_settings(
expand_wildcards="open", ignore_indices="", flat_settings=False, ignore_unavailable=False, local=True
)
self.assertIn(self._index, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.get_settings(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.get_settings(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_put_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.put_settings({"index": {"number_of_replicas": 2}}, self._index)
self.assertTrue(data["acknowledged"], data)
with self.assertRaises(RequestError):
yield from self.cl.indices.put_settings(
{"index": {"number_of_replicas": 2}},
allow_no_indices=True,
expand_wildcards="open",
flat_settings=False,
ignore_unavailable=False,
master_timeout="1s",
)
self.assertTrue(data["acknowledged"], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.put_settings({}, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.put_settings({}, expand_wildcards="1")
self.loop.run_until_complete(go())
def test_status(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.status()
self.assertIn("indices", data)
data = yield from self.cl.indices.status(
ignore_indices="",
allow_no_indices=True,
recovery=False,
snapshot=False,
operation_threading="",
expand_wildcards="open",
ignore_unavailable=False,
human=True,
)
self.assertIn("_shards", data)
with self.assertRaises(TypeError):
yield from self.cl.indices.status(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.status(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.stats()
self.assertIn("indices", data, data)
data = yield from self.cl.indices.stats(
metric="_all",
completion_fields="*",
docs=1,
fielddata_fields="*",
fields="*",
groups="*",
allow_no_indices=True,
expand_wildcards="open",
ignore_indices=False,
ignore_unavailable=True,
level="cluster",
types="*",
human=True,
)
self.assertIn("_all", data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(expand_wildcards="1")
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(level=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(level="1")
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(metric=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(metric="1")
self.loop.run_until_complete(go())
def test_segments(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.segments()
self.assertIn("indices", data, data)
self.assertIn("_shards", data, data)
data = yield from self.cl.indices.segments(
allow_no_indices=True, ignore_indices=True, ignore_unavailable=True, expand_wildcards="open", human=True
)
self.assertIn("indices", data, data)
self.assertIn("_shards", data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.segments(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.segments(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_optimize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.optimize()
self.assertIn("_shards", data)
data = yield from self.cl.indices.optimize(
allow_no_indices=True,
expand_wildcards="open",
ignore_indices=True,
ignore_unavailable=True,
max_num_segments=0,
only_expunge_deletes=True,
operation_threading="",
wait_for_merge=False,
force=True,
flush=True,
)
self.assertIn("_shards", data)
with self.assertRaises(TypeError):
yield from self.cl.indices.optimize(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.optimize(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_validate_query(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.validate_query()
self.assertIn("_shards", data)
yield from self.cl.indices.validate_query(
explain=True,
allow_no_indices=True,
q="",
ignore_indices=True,
source="",
operation_threading="",
expand_wildcards="open",
ignore_unavailable=False,
)
with self.assertRaises(TypeError):
yield from self.cl.indices.validate_query(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.validate_query(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_clear_cache(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.clear_cache()
self.assertIn("_shards", data)
yield from self.cl.indices.clear_cache(
field_data=True,
fielddata=True,
recycler=True,
id_cache=True,
filter_keys="",
filter_cache=True,
filter=False,
fields="",
id=False,
allow_no_indices=False,
ignore_indices=False,
ignore_unavailable=True,
expand_wildcards="open",
)
self.assertIn("_shards", data)
with self.assertRaises(TypeError):
yield from self.cl.indices.clear_cache(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.clear_cache(expand_wildcards="1")
self.loop.run_until_complete(go())
def test_recovery(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, "type", MESSAGE, "1")
data = yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.recovery()
self.assertIn(self._index, data)
data = yield from self.cl.indices.recovery(active_only=False, detailed=True, human=True)
self.loop.run_until_complete(go())
def test_mapping(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
mapping = {"testdoc": {"properties": {"message": {"type": "string"}}}}
# PUT
data = yield from self.cl.indices.put_mapping(self._index, "testdoc", mapping)
self.assertTrue(data["acknowledged"])
# GET
data = yield from self.cl.indices.get_mapping(self._index, "testdoc")
self.assertEqual(data["elastic_search"]["mappings"], mapping)
# DELETE
yield from self.cl.indices.delete_mapping(self._index, "testdoc")
data = yield from self.cl.indices.get_mapping(self._index, "testdoc")
self.assertFalse(data)
self.loop.run_until_complete(go())
def test_get_field_mapping(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, "type", MESSAGE, "1")
rt = yield from self.cl.indices.get_field_mapping("message", index=self._index)
self.assertEqual(
# dude, you are so deep
rt[self._index]["mappings"]["type"]["message"]["mapping"],
{"message": {"type": "string"}},
)
self.loop.run_until_complete(go())
def test_warmers(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, "type", MESSAGE, "1")
a = yield from self.cl.indices.get_warmer(name="warmer")
self.assertFalse(a)
b = {"query": {"match_all": {}}, "aggs": {"aggs_1": {"terms": {"field": "message"}}}}
yield from self.cl.indices.put_warmer(index=self._index, name="warmer", body=b)
a = yield from self.cl.indices.get_warmer(name="warmer")
self.assertIn("warmer", a[self._index]["warmers"].keys())
yield from self.cl.indices.delete_warmer(name="warmer", index=self._index)
a = yield from self.cl.indices.get_warmer(name="warmer")
self.assertFalse(a)
self.loop.run_until_complete(go())
def test_aliases(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, "type", MESSAGE, "1")
al = yield from self.cl.indices.exists_alias("alias")
self.assertFalse(al)
al = yield from self.cl.indices.get_alias("alias")
self.assertEqual({}, al)
al = yield from self.cl.indices.get_aliases("alias")
self.assertEqual({}, al)
yield from self.cl.indices.put_alias("alias", self._index)
al = yield from self.cl.indices.exists_alias("alias")
self.assertTrue(al)
yield from self.cl.indices.update_aliases(
body={
"actions": [
{"remove": {"index": self._index, "alias": "alias"}},
{"add": {"index": self._index, "alias": "alias2"}},
]
}
)
al = yield from self.cl.indices.exists_alias("alias2")
self.assertTrue(al)
yield from self.cl.indices.delete_alias(self._index, "alias2")
al = yield from self.cl.indices.get_aliases("alias")
self.assertFalse(al)
self.loop.run_until_complete(go())
def test_templates(self):
@asyncio.coroutine
def go():
b = {"template": self._index, "settings": {"number_of_shards": "1"}}
t = yield from self.cl.indices.exists_template("template")
self.assertFalse(t)
yield from self.cl.indices.put_template("template", b)
t = yield from self.cl.indices.exists_template("template")
self.assertTrue(t)
t = yield from self.cl.indices.get_template("template")
self.assertEqual(t["template"]["settings"]["index.number_of_shards"], b["settings"]["number_of_shards"])
yield from self.cl.indices.delete_template("template")
t = yield from self.cl.indices.exists_template("template")
self.assertFalse(t)
self.loop.run_until_complete(go())
def insert_topics(data):
#es = Elasticsearch(['%s:%d'%(es_ip, es_port)])
some_bulks = ''
bulk_result = 0
#bulk_result = None
try:
result = teaclient.request(data)
#print(result)
root = et.fromstring(result)
status = root.findall(
"./results/result[@name='status']")[0].text if len(
root.findall("./results/result[@name='status']")) > 0 else ''
#print(">>> Tea client response : %s" % status)
if status == "success" and len(
root.findall("./results/result[@name='keywords']")) > 0:
result_scd = root.findall(
"./results/result[@name='keywords']")[0].text
terms = ""
verbs = ""
for line in result_scd.split("\n"):
if line.startswith("<TERMS>"):
terms = line.replace("<TERMS>", "") # 하늘:387^테스트:14^도움:11
elif line.startswith("<VERBS>"):
verbs = line.replace("<VERBS>", "") # 하늘:387^테스트:14^도움:11
#print("### terms : %s" % terms)
# <TERMS>
#t = asyncio.ensure_future(time_log())
terms = [('NN', term)
for term in terms.split(teaclient.ITEM_DELIMITER)]
verbs = [('VV', verb)
for verb in verbs.split(teaclient.ITEM_DELIMITER)]
# 2018.03.26 terms와 verbs에 모두 등장하면 명사형으로 간주.
newDict = {}
for cl, topic in terms + verbs:
if topic in newDict:
newDict[topic]['cnt'] += 1
else:
newDict[topic] = {'cnt': 1, 'topic_class': cl}
newArr = []
for x in newDict.items():
if x[1]['cnt'] > 1 or x[1]['topic_class'] == 'NN':
newArr.append(('NN', x[0]))
else:
newArr.append(('VV', x[0]))
from elasticsearch import Elasticsearch
es_client = Elasticsearch(":".join([es_ip, str(es_port)]))
try:
#fts = [ make_bulk(t, data) for t in (terms+verbs) ]
fts = [make_bulk(t, data) for t in (newArr)]
#t.cancel()
some_bulks = yield from asyncio.gather(*fts)
'''
thisBulk = [
[
{'update' : { '_index' : 'topics-2018.01.01', '_type' : 'doc', ....... },
{'topic' : '증권', 'topic_id' : ..... }
],
[
{'update' : { '_index' : 'topics-2018.01.01', '_type' : 'doc', ....... },
{'topic' : '은행', 'topic_id' : ..... },
]
...
]
thisBulk = yield from asyncio.gather(*fts)
some_bulks = [ y for x in thisBulk for y in x ]
'''
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0, some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
except EsError as e:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" % (str(e), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
break
except EsError as e:
logger.error("[insert_topics] %s (retry:%d)" %
(str(e), retry))
continue
except exceptions.ConnectionTimeout as timeoutError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(timeoutError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
break
except exceptions.ConnectionTimeout as timeoutError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(timeoutError), retry))
continue
except aiohttp.client_exceptions.ClientConnectorError as connectError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
break
except aiohttp.client_exceptions.ClientConnectorError as connectError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectError), retry))
continue
except OSError as oserror:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(oserror), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
break
except OSError as oserror:
logger.error("[insert_topics] %s (retry:%d)" %
(str(oserror), retry))
continue
except urllib3.exceptions.NewConnectionError as connectionError:
retry = 0
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectionError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)),
refresh=True)[0]
#bulk_result = yield from es.bulk(filter(lambda x:x and len(x)>0, some_bulks))
break
except urllib3.exceptions.NewConnectionError as connectionError:
logger.error("[insert_topics] %s (retry:%d)" %
(str(connectionError), retry))
continue
except:
ex = traceback.format_exc()
logger.error(
"[insert_topics] unknown error. Traceback >> %s " % ex)
logger.debug("%d are successfully inserted." % bulk_result)
#logger.debug("%d are successfully inserted."%len(bulk_result['items']))
except ParseError as xmlerror:
logger.error("[insert_topics] TeaClient failed. (%s)" % str(xmlerror))
logger.error("==============> teaclient's xml response : %s" % result)
class TestCat(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_aliases(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.aliases(v=True)
self.assertIn('alias', ret)
self.assertIn('index', ret)
self.loop.run_until_complete(go())
def test_allocation(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.allocation(v=True)
self.assertIn('disk.percent', ret)
self.loop.run_until_complete(go())
def test_count(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.count(v=True)
self.assertIn('timestamp', ret)
self.assertIn('count', ret)
# testing for index
yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
},
'1'
)
ret = yield from self.cl.cat.count(self._index, v=True)
self.assertIn('timestamp', ret)
self.assertIn('count', ret)
self.loop.run_until_complete(go())
def test_health(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.health(v=True)
self.assertIn('timestamp', ret)
self.assertIn('node.total', ret)
self.loop.run_until_complete(go())
def test_help(self):
pattern = textwrap.dedent("""\
=^.^=
/_cat/allocation
/_cat/shards
/_cat/shards/{index}
/_cat/master
/_cat/nodes
/_cat/indices
/_cat/indices/{index}
/_cat/segments
/_cat/segments/{index}
/_cat/count
/_cat/count/{index}
/_cat/recovery
/_cat/recovery/{index}
/_cat/health
/_cat/pending_tasks
/_cat/aliases
/_cat/aliases/{alias}
/_cat/thread_pool
/_cat/plugins
/_cat/fielddata
/_cat/fielddata/{fields}
""")
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.help(help=True)
self.assertEqual(pattern, ret)
self.loop.run_until_complete(go())
def test_indices(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.indices(v=True)
self.assertIn('health', ret)
self.assertIn('index', ret)
self.loop.run_until_complete(go())
def test_master(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.master(v=True)
self.assertIn('host', ret)
self.assertIn('ip', ret)
self.loop.run_until_complete(go())
def test_nodes(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.nodes(v=True)
self.assertIn('load', ret)
self.assertIn('name', ret)
self.loop.run_until_complete(go())
def test_recovery(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.recovery(v=True)
self.assertIn('index', ret)
self.assertIn('files', ret)
self.loop.run_until_complete(go())
def test_shards(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.shards(v=True)
self.assertIn('index', ret)
self.assertIn('node', ret)
self.loop.run_until_complete(go())
def test_segments(self):
@asyncio.coroutine
def go():
yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
},
'1'
)
ret = yield from self.cl.cat.segments(index=self._index, v=True)
self.assertIn('index', ret)
self.assertIn('segment', ret)
self.loop.run_until_complete(go())
def test_pending_tasks(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.pending_tasks(v=True)
self.assertIn('insertOrder', ret)
self.assertIn('priority', ret)
self.loop.run_until_complete(go())
def test_thread_pool(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.thread_pool(v=True)
self.assertIn('host', ret)
self.assertIn('ip', ret)
self.loop.run_until_complete(go())
def test_fielddata(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.fielddata(v=True)
self.assertIn('id', ret)
self.assertIn('total', ret)
self.loop.run_until_complete(go())
def test_plugins(self):
@asyncio.coroutine
def go():
ret = yield from self.cl.cat.plugins(v=True)
self.assertIn('name', ret)
self.assertIn('component', ret)
self.loop.run_until_complete(go())
class ElasticsearchController:
def __init__(self):
self._elasticsearch = None
self.elasticsearch_conn = False
self._loop = None
def get_loop(self):
return self._loop
def get_elasticsearch_client(self):
return self._elasticsearch_client
@timestamped
async def get_value(self, index: str, id: str, doc_type='_all'):
"""
Get record from elasticsearch using id and index
:param index: used to search the named index for records
:param id: used to find record in the index
:param doc_type:
:return: []
"""
try:
assert await self.exists(index, id) is True
result = await self._elasticsearch_client.get(index, id)
except AssertionError as e:
logger.error(
"Elasticsearch client doesn't exist when it should. " + str(e))
result = ""
result = result.decode('utf-8')
return result
@timestamped
async def create(self, index, doc_type, body, id=None):
"""
Used to create new record in the elasticsearch database
:param index: used to create in that specific index
:param doc_type: specify elasticsearch document type
:param body: actual body of the record to be created in the database
:return: Json object
"""
return await self._elasticsearch_client.create(index,
doc_type,
body,
id=42)
async def exists(self, index, id):
"""
Used to check if record exist in the elasticsearch database using id and index
:param index: search index for record
:param id: find record for the id
:return:
"""
return await self._elasticsearch_client.exists(index, id)
@timestamped
async def update(self, index, doc_type, id, body=None):
"""
Used to update record in the elasticsearch database
:param index: select index to update record in that index
:param doc_type: specify elasticsearch document type
:param id: Identifier for the record to be updated
:param body: the actual body for the record
:return:
"""
return await self._elasticsearch_client.update(index,
doc_type,
id,
body=body)
@timestamped
async def search(self, index=None, doc_type=None, body=None):
"""
Used to search for the record in the elasticsearch database
:param index: used to search the index
:param doc_type:
:param body: query to be executed to match the result
:return:
"""
return await self._elasticsearch_client.search(index=index,
doc_type=doc_type,
body=body)
@timestamped
async def delete(self, index, doc_type, id):
"""
Used to delete record from elasticsearch database
:param index: specify the index for the record to be deleted
:param doc_type:
:param id: specify the id for the record to be deleted
:return:
"""
return await self._elasticsearch_client.delete(index, doc_type, id)
def set_loop(self, loop):
self._loop = loop
@timestamped
async def add_elasticsearch_connection(self):
elasticsearch_host = None
elasticsearch_port = None
try:
elasticsearch_host = os.getenv("ELASTICSEARCH_HOST",
"redis.dev.muchneededllc.com")
elasticsearch_port = str(os.getenv("ELASTICSEARCH_PORT", 9200))
except OSError as e:
logger.error(
"Couldn't get environmental variables for elasticearch. " +
str(e))
exit(1)
try:
if self._loop is not None:
address = ':'.join([elasticsearch_host, elasticsearch_port])
self._elasticsearch_client = Elasticsearch([address],
loop=self._loop)
logger.info(self._elasticsearch_client)
self.elasticsearch_conn = True
logger.debug("Created Elasticsearch Client.")
else:
logger.error(
"Couldn't create elasticsearch client because loop hasn't been set."
)
except Exception as e:
logger.error("couldn't open elasticsearch.")
raise Exception(e)
def cleanup(self):
if self.elasticsearch_conn:
self._elasticsearch_client.close()
class ElasticSearchManager(DefaultSearchUtility):
def __init__(self, settings={}, loop=None):
self.loop = loop
self._conn = None
self._migration_lock = None
@property
def bulk_size(self):
return self.settings.get('bulk_size', 50)
@property
def settings(self):
return app_settings.get('elasticsearch', {})
@property
def conn(self):
if self._conn is None:
self._conn = Elasticsearch(loop=self.loop,
**self.settings['connection_settings'])
return self._conn
@property
def enabled(self):
return len(
self.settings.get('connection_settings', {}).get('endpoints',
[])) > 0
async def initialize(self, app):
self.app = app
self._migration_lock = asyncio.Lock()
async def finalize(self, app):
if self._conn is not None:
self._conn.close()
async def get_registry(self, container, request):
if request is None:
request = get_current_request()
if hasattr(request, 'container_settings'):
return request.container_settings
annotations_container = IAnnotations(container)
request.container_settings = await annotations_container.async_get(
REGISTRY_DATA_KEY)
return request.container_settings
async def get_real_index_name(self, container, request=None):
index_name = await self.get_index_name(container, request)
version = await self.get_version(container, request)
return index_name + '_' + str(version)
async def get_index_name(self, container, request=None):
registry = await self.get_registry(container, request)
try:
result = registry['el_index_name']
except KeyError:
result = app_settings['elasticsearch'].get(
'index_name_prefix', 'guillotina-') + container.id
return result
async def get_next_index_name(self, container, request=None):
registry = await self.get_registry(container, request)
if ('el_next_index_version' not in registry
or registry['el_next_index_version'] is None):
return None
index_name = await self.get_index_name(container, request)
version = registry['el_next_index_version']
return index_name + '_' + str(version)
async def set_index_name(self, container, name, request=None):
registry = await self.get_registry(container, request)
registry['el_index_name'] = name
registry._p_register()
async def initialize_catalog(self, container):
if not self.enabled:
return
await self.remove_catalog(container)
index_name = await self.get_index_name(container)
real_index_name = await self.get_real_index_name(container)
await safe_es_call(self.conn.indices.create, real_index_name)
await safe_es_call(self.conn.indices.put_alias, index_name,
real_index_name)
await safe_es_call(self.conn.indices.close, index_name)
await safe_es_call(self.install_mappings_on_index, index_name)
await self.conn.indices.open(index_name)
await self.conn.cluster.health(wait_for_status='yellow')
await self.set_index_name(container, index_name)
async def remove_catalog(self, container):
if not self.enabled:
return
index_name = await self.get_index_name(container)
real_index_name = await self.get_real_index_name(container)
await safe_es_call(self.conn.indices.close, real_index_name)
await safe_es_call(self.conn.indices.delete_alias, real_index_name,
index_name)
await safe_es_call(self.conn.indices.delete, real_index_name)
await safe_es_call(self.conn.indices.delete, index_name)
async def get_version(self, container, request=None):
registry = await self.get_registry(container, request)
try:
version = registry['el_index_version']
except KeyError:
version = 1
return version
async def set_version(self, container, version, request=None, force=False):
registry = await self.get_registry(container, request)
if (not force and 'el_next_index_version' in registry
and registry['el_next_index_version'] is not None):
raise Exception(
'Cannot change index while migration is in progress')
registry['el_index_version'] = version
registry._p_register()
async def stats(self, container):
index_name = await self.get_index_name(container)
return await self.conn.indices.stats(index_name)
async def install_mappings_on_index(self, index_name):
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
await self.conn.indices.close(index_name)
await self.conn.indices.put_settings(index_settings, index_name)
for key, value in mappings.items():
await self.conn.indices.put_mapping(index_name, key, value)
await self.conn.indices.open(index_name)
async def activate_next_index(self,
container,
version,
request=None,
force=False):
'''
Next index support designates an index to also push
delete and index calls to
'''
registry = await self.get_registry(container, request)
if not force:
try:
assert registry['el_next_index_version'] is None
except KeyError:
pass
registry['el_next_index_version'] = version
registry._p_register()
async def disable_next_index(self, container, request=None):
'''
Next index support designates an index to also push
delete and index calls to
'''
registry = await self.get_registry(container, request)
registry['el_next_index_version'] = None
registry._p_register()
async def apply_next_index(self, container, request=None):
# make sure to reload the registry to make sure we have the latest
# to write to
if (request is not None and hasattr(request, 'container_settings')
and REGISTRY_DATA_KEY in container.__annotations__):
await request._txn.refresh(request.container_settings)
registry = await self.get_registry(container, request)
assert registry['el_next_index_version'] is not None
await self.set_version(container,
registry['el_next_index_version'],
request,
force=True)
registry['el_next_index_version'] = None
registry._p_register()
class TestIndices(unittest.TestCase):
def setUp(self):
self._index = 'elastic_search'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(
self.cl.delete(self._index, refresh=True))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_analize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.analyze(
text='this, is a test 125 !',
analyzer='standard',
filters='lowercase')
self.assertEqual(len(data['tokens']), 5)
self.assertEqual(data['tokens'][0]['token'], 'this')
self.assertEqual(data['tokens'][1]['token'], 'is')
self.assertEqual(data['tokens'][2]['token'], 'a')
self.assertEqual(data['tokens'][3]['token'], 'test')
self.assertEqual(data['tokens'][4]['token'], '125')
self.assertEqual(data['tokens'][4]['type'], '<NUM>')
data = yield from self.cl.indices.analyze(
text='this is a <b>test</b>',
tokenizer='keyword',
token_filters='lowercase',
char_filters='html_strip',
prefer_local=True)
self.assertEqual(data['tokens'][0]['token'], 'this is a test')
with self.assertRaises(RequestError):
yield from self.cl.indices.analyze(analyzer='standard',
filters='lowercase',
field='w')
self.loop.run_until_complete(go())
def test_create(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.create(self._index,
timeout=1000,
master_timeout=1000)
self.assertTrue(data['acknowledged'])
self.loop.run_until_complete(go())
def test_refresh(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.refresh(self._index)
self.assertIn('_shards', data, data)
yield from self.cl.indices.refresh(self._index,
allow_no_indices=False,
expand_wildcards='closed',
ignore_unavailable=True,
ignore_indices='',
force=True)
with self.assertRaises(TypeError):
yield from self.cl.indices.refresh(self._index,
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.refresh(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_flush(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.flush(self._index)
self.assertIn('_shards', data, data)
yield from self.cl.indices.flush(self._index,
full=True,
allow_no_indices=False,
expand_wildcards='closed',
ignore_unavailable=True,
ignore_indices='',
force=True)
with self.assertRaises(TypeError):
yield from self.cl.indices.flush(self._index,
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.flush(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_open(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
data = yield from self.cl.indices.open(self._index,
timeout=1000,
master_timeout=1000,
allow_no_indices=False,
expand_wildcards='closed',
ignore_unavailable=True)
self.assertTrue(data['acknowledged'], data)
data = yield from self.cl.indices.open(self._index)
with self.assertRaises(TypeError):
yield from self.cl.indices.open(self._index,
expand_wildcards=1,
ignore_unavailable=True)
with self.assertRaises(ValueError):
yield from self.cl.indices.open(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_close(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.cluster.health(self._index,
wait_for_status='yellow')
data = yield from self.cl.indices.close(self._index)
self.assertTrue(data['acknowledged'], data)
data = yield from self.cl.indices.close(self._index,
timeout='1s',
master_timeout='1s',
expand_wildcards='open',
allow_no_indices=True,
ignore_unavailable=True)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.close(self._index,
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.close(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_delete(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.delete(self._index)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(NotFoundError):
yield from self.cl.indices.delete(self._index,
timeout='1s',
master_timeout='1s')
self.assertTrue(data['acknowledged'], data)
self.loop.run_until_complete(go())
def test_exists(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.exists(self._index,
allow_no_indices=False,
expand_wildcards='closed',
ignore_unavailable=False,
local=False)
self.assertTrue(data)
data = yield from self.cl.indices.exists(self._index + '123')
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists(self._index,
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists(self._index,
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_exists_type(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.exists_type(
self._index, 'type', allow_no_indices=False)
self.assertTrue(data)
data = yield from self.cl.indices.exists_type(
self._index, 'ert', expand_wildcards='open')
self.assertFalse(data)
with self.assertRaises(TypeError):
yield from self.cl.indices.exists_type(self._index,
'',
expand_wildcards=1,
allow_no_indices=True,
ignore_unavailable=True,
ignore_indices=True,
local=True)
with self.assertRaises(ValueError):
yield from self.cl.indices.exists_type(self._index,
'',
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_get_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.get_settings()
self.assertIn(self._index, data)
data = yield from self.cl.indices.get_settings(
expand_wildcards='open',
ignore_indices='',
flat_settings=False,
ignore_unavailable=False,
local=True)
self.assertIn(self._index, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.get_settings(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.get_settings(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_put_settings(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.put_settings(
{"index": {
"number_of_replicas": 2
}}, self._index)
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(RequestError):
yield from self.cl.indices.put_settings(
{"index": {
"number_of_replicas": 2
}},
allow_no_indices=True,
expand_wildcards='open',
flat_settings=False,
ignore_unavailable=False,
master_timeout='1s')
self.assertTrue(data['acknowledged'], data)
with self.assertRaises(TypeError):
yield from self.cl.indices.put_settings({}, expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.put_settings({},
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_status(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.status()
self.assertIn('indices', data)
data = yield from self.cl.indices.status(ignore_indices='',
allow_no_indices=True,
recovery=False,
snapshot=False,
operation_threading='',
expand_wildcards='open',
ignore_unavailable=False,
human=True)
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.status(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.status(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_stats(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.stats()
self.assertIn('indices', data, data)
data = yield from self.cl.indices.stats(metric='_all',
completion_fields='*',
docs=1,
fielddata_fields='*',
fields='*',
groups='*',
allow_no_indices=True,
expand_wildcards='open',
ignore_indices=False,
ignore_unavailable=True,
level='cluster',
types='*',
human=True)
self.assertIn('_all', data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(expand_wildcards='1')
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(level=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(level='1')
with self.assertRaises(TypeError):
yield from self.cl.indices.stats(metric=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.stats(metric='1')
self.loop.run_until_complete(go())
def test_segments(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.segments()
self.assertIn('indices', data, data)
self.assertIn('_shards', data, data)
data = yield from self.cl.indices.segments(allow_no_indices=True,
ignore_indices=True,
ignore_unavailable=True,
expand_wildcards='open',
human=True)
self.assertIn('indices', data, data)
self.assertIn('_shards', data, data)
with self.assertRaises(TypeError):
yield from self.cl.indices.segments(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.segments(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_optimize(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.optimize()
self.assertIn('_shards', data)
data = yield from self.cl.indices.optimize(
allow_no_indices=True,
expand_wildcards='open',
ignore_indices=True,
ignore_unavailable=True,
max_num_segments=0,
only_expunge_deletes=True,
operation_threading='',
wait_for_merge=False,
force=True,
flush=True)
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.optimize(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.optimize(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_validate_query(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.validate_query()
self.assertIn('_shards', data)
yield from self.cl.indices.validate_query(explain=True,
allow_no_indices=True,
q='',
ignore_indices=True,
source='',
operation_threading='',
expand_wildcards='open',
ignore_unavailable=False)
with self.assertRaises(TypeError):
yield from self.cl.indices.validate_query(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.validate_query(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_clear_cache(self):
@asyncio.coroutine
def go():
data = yield from self.cl.indices.clear_cache()
self.assertIn('_shards', data)
yield from self.cl.indices.clear_cache(field_data=True,
fielddata=True,
recycler=True,
id_cache=True,
filter_keys='',
filter_cache=True,
filter=False,
fields='',
id=False,
allow_no_indices=False,
ignore_indices=False,
ignore_unavailable=True,
expand_wildcards='open')
self.assertIn('_shards', data)
with self.assertRaises(TypeError):
yield from self.cl.indices.clear_cache(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.indices.clear_cache(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_recovery(self):
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
data = yield from self.cl.indices.refresh(self._index)
data = yield from self.cl.indices.recovery()
self.assertIn(self._index, data)
data = yield from self.cl.indices.recovery(active_only=False,
detailed=True,
human=True)
self.loop.run_until_complete(go())
def test_mapping(self):
@asyncio.coroutine
def go():
yield from self.cl.indices.create(self._index)
mapping = {
'testdoc': {
'properties': {
'message': {
'type': 'string',
}
}
}
}
# PUT
data = yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
self.assertTrue(data['acknowledged'])
# GET
data = yield from self.cl.indices.get_mapping(
self._index,
'testdoc',
)
self.assertEqual(data['elastic_search']['mappings'], mapping)
# DELETE
yield from self.cl.indices.delete_mapping(
self._index,
'testdoc',
)
data = yield from self.cl.indices.get_mapping(
self._index,
'testdoc',
)
self.assertFalse(data)
self.loop.run_until_complete(go())
def test_get_field_mapping(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
rt = yield from self.cl.indices.get_field_mapping(
'message', index=self._index)
self.assertEqual(
# dude, you are so deep
rt[self._index]['mappings']['type']['message']['mapping'],
{'message': {
'type': 'string'
}})
self.loop.run_until_complete(go())
def test_warmers(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertFalse(a)
b = {
"query": {
"match_all": {}
},
"aggs": {
"aggs_1": {
"terms": {
"field": "message"
}
}
}
}
yield from self.cl.indices.put_warmer(index=self._index,
name='warmer',
body=b)
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertIn('warmer', a[self._index]['warmers'].keys())
yield from self.cl.indices.delete_warmer(name='warmer',
index=self._index)
a = yield from self.cl.indices.get_warmer(name='warmer')
self.assertFalse(a)
self.loop.run_until_complete(go())
def test_aliases(self):
@asyncio.coroutine
def go():
# create index
yield from self.cl.index(self._index, 'type', MESSAGE, '1')
al = yield from self.cl.indices.exists_alias('alias')
self.assertFalse(al)
al = yield from self.cl.indices.get_alias('alias')
self.assertEqual({}, al)
al = yield from self.cl.indices.get_aliases('alias')
self.assertEqual({}, al)
yield from self.cl.indices.put_alias('alias', self._index)
al = yield from self.cl.indices.exists_alias('alias')
self.assertTrue(al)
yield from self.cl.indices.update_aliases(
body={
"actions": [{
"remove": {
"index": self._index,
"alias": "alias"
}
}, {
"add": {
"index": self._index,
"alias": "alias2"
}
}]
})
al = yield from self.cl.indices.exists_alias('alias2')
self.assertTrue(al)
yield from self.cl.indices.delete_alias(self._index, 'alias2')
al = yield from self.cl.indices.get_aliases('alias')
self.assertFalse(al)
self.loop.run_until_complete(go())
def test_templates(self):
@asyncio.coroutine
def go():
b = {
"template": self._index,
"settings": {
"number_of_shards": '1'
},
}
t = yield from self.cl.indices.exists_template('template')
self.assertFalse(t)
yield from self.cl.indices.put_template('template', b)
t = yield from self.cl.indices.exists_template('template')
self.assertTrue(t)
t = yield from self.cl.indices.get_template('template')
self.assertEqual(
t['template']['settings']['index.number_of_shards'],
b['settings']['number_of_shards'])
yield from self.cl.indices.delete_template('template')
t = yield from self.cl.indices.exists_template('template')
self.assertFalse(t)
self.loop.run_until_complete(go())
class TestClient(unittest.TestCase):
def setUp(self):
self._index = 'test_elasticsearch'
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.cl = Elasticsearch([{'host': 'localhost'}], loop=self.loop)
self.addCleanup(self.cl.close)
try:
self.loop.run_until_complete(self.cl.delete(self._index, '', ''))
except NotFoundError:
pass
def tearDown(self):
self.loop.close()
def test_ping(self):
""" ping """
class R:
@asyncio.coroutine
def perform_request(self, a, b):
yield
raise TransportError
def close(self):
pass
@asyncio.coroutine
def go():
data = yield from self.cl.ping()
self.assertTrue(data)
self.cl._transport = R()
yield from self.cl.ping()
self.loop.run_until_complete(go())
self.cl.__repr__()
def test_info(self):
""" test_info """
@asyncio.coroutine
def go():
data = yield from self.cl.info()
self.assertEqual(data['status'], 200)
self.loop.run_until_complete(go())
def test_create(self):
""" create index """
@asyncio.coroutine
def go():
data = yield from self.cl.create(
self._index, 'tweet',
{
'user': '******',
'skills': ['C', 'Python', 'Assembler'],
'date': '2009-11-15T14:12:12'
},
'1',
routing='Bob')
self.assertEqual(data['_index'], self._index)
self.assertEqual(data['_type'], 'tweet')
self.assertEqual(data['_version'], 1)
self.assertTrue(data['created'], data)
# test for conflict
with self.assertRaises(ConflictError):
yield from self.cl.create(
self._index, 'tweet', {}, '1')
self.loop.run_until_complete(go())
def test_index(self):
""" auto-create index """
@asyncio.coroutine
def go():
data = yield from self.cl.index(self._index, 'tweet', {}, '1')
self.assertEqual(data['_index'], self._index)
self.assertEqual(data['_type'], 'tweet')
self.assertEqual(data['_id'], '1')
self.assertEqual(data['_version'], 1)
self.assertTrue(data['created'], data)
# test increment version
data = yield from self.cl.index(self._index, 'tweet', {}, '1')
self.assertEqual(data['_version'], 2)
self.assertFalse(data['created'], data)
# test 'external' version_type
data = yield from self.cl.index(self._index, 'tweet', {}, '12',
version_type='external',
version=122,
timestamp='2009-11-15T14:12:12',
ttl='1d',
consistency='one',
timeout=30000,
refresh=True,
replication='async')
self.assertEqual(data['_version'], 122)
self.assertTrue(data['created'], data)
with self.assertRaises(RequestError):
yield from self.cl.index(self._index, 'type', {},
parent='1',
percolate='')
with self.assertRaises(TypeError):
yield from self.cl.index(self._index, 'type', {},
consistency=1)
with self.assertRaises(ValueError):
yield from self.cl.index(self._index, 'type', {},
consistency='1')
with self.assertRaises(TypeError):
yield from self.cl.index(self._index, 'type', {},
replication=1)
with self.assertRaises(ValueError):
yield from self.cl.index(self._index, 'type', {},
replication='1')
with self.assertRaises(TypeError):
yield from self.cl.index(self._index, 'type', {},
op_type=1)
with self.assertRaises(ValueError):
yield from self.cl.index(self._index, 'type', {},
op_type='1')
with self.assertRaises(TypeError):
yield from self.cl.index(self._index, 'tweet', {},
version_type=1)
with self.assertRaises(ValueError):
yield from self.cl.index(self._index, 'tweet', {},
version_type='1')
self.loop.run_until_complete(go())
def test_exist(self):
""" exists """
@asyncio.coroutine
def go():
id = '100'
# test non-exist
data = yield from self.cl.exists(self._index, id,
refresh=True,
realtime=True,
preference='_local')
self.assertFalse(data)
# test exist
yield from self.cl.index(self._index, 'exist',
{'user': '******', 'tim': 'none'},
id,
routing='opa')
data = yield from self.cl.exists(self._index, id,
routing='opa')
self.assertTrue(data)
data = yield from self.cl.exists(self._index, id, parent='1')
self.assertFalse(data, data)
self.loop.run_until_complete(go())
def test_get(self):
""" get """
@asyncio.coroutine
def go():
id = '200'
yield from self.cl.index(self._index, 'test_get', MESSAGES[1], id)
data = yield from self.cl.get(self._index, id,
realtime=True,
refresh=True)
self.assertEqual(data['_id'], id)
self.assertEqual(data['_index'], self._index)
self.assertEqual(data['_type'], 'test_get')
self.assertEqual(data['_version'], 1)
self.assertTrue(data['found'], data)
self.assertEqual(data['_source'], MESSAGES[1])
data = yield from self.cl.get(self._index, id,
_source=False)
self.assertEqual(data['_source'], {})
data = yield from self.cl.get(self._index, id,
_source_exclude='counter',
_source_include='*')
self.assertNotIn('counter', data, data)
data = yield from self.cl.get(self._index, id,
fields='user,skills',
routing='Sidor',
preference='_local',
version=1,
version_type='internal')
self.assertIn('fields', data, data)
self.assertIn('user', data['fields'], data)
self.assertIn('skills', data['fields'], data)
with self.assertRaises(NotFoundError):
yield from self.cl.get(self._index, id, parent='1')
with self.assertRaises(TypeError):
yield from self.cl.get(self._index, id,
version_type=1)
with self.assertRaises(ValueError):
yield from self.cl.get(self._index, id,
version_type='1')
self.loop.run_until_complete(go())
def test_get_source(self):
""" get_source """
@asyncio.coroutine
def go():
yield from self.cl.index(self._index,
'test_get_source',
MESSAGES[0],
'1')
data = yield from self.cl.get_source(self._index, '1')
self.assertEqual(data, MESSAGES[0])
id = '200'
yield from self.cl.index(
self._index, 'test_get_source', MESSAGES[2], id,
routing='Poligrafovich'
)
data = yield from self.cl.get_source(self._index, id,
routing='Poligrafovich',
preference='_local',
version=1,
version_type='internal',
realtime=True,
refresh=True)
self.assertEqual(data, MESSAGES[2])
data = yield from self.cl.get_source(self._index, id,
routing='Poligrafovich',
_source=False)
self.assertEqual(data, {})
data = yield from self.cl.get_source(self._index, id,
routing='Poligrafovich',
_source_exclude='counter',
_source_include='*')
self.assertNotIn('counter', data, data)
self.assertTrue('user', data)
with self.assertRaises(NotFoundError):
yield from self.cl.get_source(self._index, id, parent='1')
with self.assertRaises(TypeError):
yield from self.cl.get_source(self._index, id,
version_type=1)
with self.assertRaises(ValueError):
yield from self.cl.get_source(self._index, id,
version_type='1')
self.loop.run_until_complete(go())
def test_delete(self):
""" delete """
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc', MESSAGES[2], '1')
data = yield from self.cl.delete(self._index, 'testdoc', '1')
self.assertTrue(data['found'], data)
with self.assertRaises(NotFoundError):
data = yield from self.cl.delete(self._index, 'testdoc', '1',
consistency='one',
replication='async',
refresh=True,
version_type='internal',
version=2,
timeout=30000,
routing='test',
parent='1')
with self.assertRaises(TypeError):
yield from self.cl.delete(self._index, 'type', {},
consistency=1)
with self.assertRaises(ValueError):
yield from self.cl.delete(self._index, 'type', {},
consistency='1')
with self.assertRaises(TypeError):
yield from self.cl.delete(self._index, 'type', {},
replication=1)
with self.assertRaises(ValueError):
yield from self.cl.delete(self._index, 'type', {},
replication='1')
with self.assertRaises(TypeError):
yield from self.cl.delete(self._index, 'type', {},
version_type=1)
with self.assertRaises(ValueError):
yield from self.cl.delete(self._index, 'type', {},
version_type='1')
self.loop.run_until_complete(go())
def test_update(self):
""" update """
@asyncio.coroutine
def go():
script = {
"doc": {
"counter": 123
}
}
yield from self.cl.index(self._index, 'testdoc', MESSAGES[2],
'1',
routing='Fedor')
yield from self.cl.update(self._index, 'testdoc', '1',
script,
version_type='internal',
version=1,
routing='Fedor')
data = yield from self.cl.get(self._index, '1', routing='Fedor')
self.assertEqual(data['_source']['counter'], 123)
self.assertEqual(data['_version'], 2)
data = yield from self.cl.update(self._index, 'testdoc', '1',
script,
timestamp='2009-11-15T14:12:12',
ttl='1d',
consistency='one',
timeout=30000,
refresh=True,
replication='async',
retry_on_conflict=2,
routing='Fedor',
lang='en')
self.assertEqual(data['_version'], 3)
with self.assertRaises(NotFoundError):
yield from self.cl.update(
self._index, 'testdoc', '1',
script={},
fields='user',
parent='1')
with self.assertRaises(TypeError):
yield from self.cl.update(self._index, 'type', {},
consistency=1)
with self.assertRaises(ValueError):
yield from self.cl.update(self._index, 'type', {},
consistency='1')
with self.assertRaises(TypeError):
yield from self.cl.update(self._index, 'type', {},
replication=1)
with self.assertRaises(ValueError):
yield from self.cl.update(self._index, 'type', {},
replication='1')
with self.assertRaises(TypeError):
yield from self.cl.update(self._index, 'type', {},
version_type=1)
with self.assertRaises(ValueError):
yield from self.cl.update(self._index, 'type', {},
version_type='1')
self.loop.run_until_complete(go())
def test_search(self):
""" search """
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[1], '2',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[2], '3',
refresh=True)
data = yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
_source=False,
_source_include='skills')
self.assertEqual(data['hits']['total'], 2, data)
self.assertIn('skills', data['hits']['hits'][0]['_source'])
self.assertIn('skills', data['hits']['hits'][1]['_source'])
data = yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
_source_exclude='skills',
analyzer='standard',
default_operator='AND',
analyze_wildcard=True,
version=2,
timeout=30000,
allow_no_indices=True,
ignore_unavailable=True,
df='_all',
explain=True,
fields='skills,user',
from_=0,
expand_wildcards='open',
lenient=True,
lowercase_expanded_terms=True,
preference='random',
scroll='1s',
search_type='query_then_fetch',
size=100,
sort='user:true',
stats=True
)
self.assertNotIn('skills', data['hits']['hits'][0]['_source'])
self.assertNotIn('skills', data['hits']['hits'][1]['_source'])
with self.assertRaises(TypeError):
yield from self.cl.search(default_operator=1,
indices_boost=False)
with self.assertRaises(ValueError):
yield from self.cl.search(doc_type='testdoc',
q='skills:Python',
routing='Sidor',
source='Query DSL',
suggest_field='user',
suggest_text='test',
suggest_mode='missing',
suggest_size=100,
default_operator='1')
with self.assertRaises(TypeError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
suggest_mode=1)
with self.assertRaises(ValueError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
suggest_mode='1')
with self.assertRaises(TypeError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
search_type=1)
with self.assertRaises(ValueError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
search_type='1')
with self.assertRaises(TypeError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.search(self._index,
'testdoc',
q='skills:Python',
expand_wildcards='1')
self.loop.run_until_complete(go())
def test_count(self):
""" count """
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[1], '2',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[2], '3',
refresh=True)
data = yield from self.cl.count(
self._index, 'testdoc', q='skills:Python')
self.assertEqual(data['count'], 2, data)
data = yield from self.cl.count(
self._index, 'testdoc', q='skills:Python',
ignore_unavailable=True,
expand_wildcards='open',
allow_no_indices=False,
min_score=1,
preference='random')
self.assertEqual(data['count'], 2, data)
with self.assertRaises(TypeError):
yield from self.cl.count(
self._index, 'testdoc',
expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.count(
self._index, 'testdoc', q='skills:Python',
expand_wildcards='1',
routing='Sidor',
source='Query DSL')
self.loop.run_until_complete(go())
def test_explain(self):
""" explain """
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[1], '2',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[2], '3',
refresh=True)
data = yield from self.cl.explain(
self._index, 'testdoc', '3',
q='skills:Python')
self.assertTrue(data['matched'], data)
data = yield from self.cl.explain(
self._index, 'testdoc', '1',
q='skills:Python',
analyze_wildcard=True,
_source=False,
_source_include='user',
_source_exclude='counter',
analyzer='standard',
default_operator='and',
df='_all',
fields='user,counter',
lenient=True,
lowercase_expanded_terms=False,
preference='random')
self.assertTrue(data['matched'], data)
with self.assertRaises(TypeError):
yield from self.cl.explain(
self._index, 'testdoc', '1',
q='skills:Python',
default_operator=1)
with self.assertRaises(ValueError):
yield from self.cl.explain(
self._index, 'testdoc', '1',
default_operator='1',
parent='2',
routing='Sidor',
source='DSL Query')
self.loop.run_until_complete(go())
def test_delete_by_query(self):
""" delete_by_query """
DQ = {"query": {"term": {"user": "******"}}}
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc', MESSAGES[3], '1')
yield from self.cl.index(self._index, 'testdoc', MESSAGES[2], '2')
# data = yield from self.cl.delete(self._index, 'testdoc', '1')
# self.assertTrue(data['found'], data)
data = yield from self.cl.delete_by_query(
self._index,
'testdoc',
q='user:Fedor Poligrafovich'
)
self.assertIn('_indices', data, data)
with self.assertRaises(TransportError):
yield from self.cl.delete_by_query(
body=DQ,
allow_no_indices=True,
analyzer='standard',
df='_all',
expand_wildcards='open',
consistency='all',
default_operator='AND',
ignore_unavailable=True,
replication='async',
routing='Fedor',
source='',
timeout='100ms')
with self.assertRaises(TypeError):
yield from self.cl.delete_by_query(default_operator=1)
with self.assertRaises(ValueError):
yield from self.cl.delete_by_query(default_operator='1')
with self.assertRaises(TypeError):
yield from self.cl.delete_by_query(consistency=1)
with self.assertRaises(ValueError):
yield from self.cl.delete_by_query(consistency='1')
with self.assertRaises(TypeError):
yield from self.cl.delete_by_query(replication=1)
with self.assertRaises(ValueError):
yield from self.cl.delete_by_query(replication='1')
with self.assertRaises(TypeError):
yield from self.cl.delete_by_query(expand_wildcards=1)
with self.assertRaises(ValueError):
yield from self.cl.delete_by_query(expand_wildcards='1')
self.loop.run_until_complete(go())
def test_msearch(self):
""" msearch """
queries = [
{"_index": self._index},
{"query": {"match_all": {}}, "from": 0, "size": 10},
{"_index": self._index},
{"query": {"match_all": {}}}
]
@asyncio.coroutine
def go():
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[1], '2',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[2], '3',
refresh=True)
data = yield from self.cl.msearch(queries)
self.assertGreater(len(data['responses']), 0, data)
data = yield from self.cl.msearch(queries, search_type='count')
self.assertGreater(len(data['responses']), 0, data)
with self.assertRaises(TypeError):
yield from self.cl.msearch(queries, search_type=1)
with self.assertRaises(ValueError):
yield from self.cl.msearch(queries, search_type='1')
self.loop.run_until_complete(go())
def test_scroll(self):
""" scroll """
@asyncio.coroutine
def go():
pass
# scroll_id = 'c2Nhbjs2OzMFXNlNyNm5JWUc1'
# with self.assertRaises(RequestError):
# yield from self.cl.scroll(scroll_id,
# scroll='1m')
# with self.assertRaises(RequestError):
# yield from self.cl.scroll(scroll_id)
self.loop.run_until_complete(go())
def test_clear_scroll(self):
""" clear_scroll """
@asyncio.coroutine
def go():
pass
# scroll_id = 'c2Nhbjs2OzMFXNlNyNm5JWUc1'
# yield from self.cl.scroll(scroll_id)
# yield from self.cl.clear_scroll(scroll_id)
self.loop.run_until_complete(go())
def test_bulk(self):
bulks = [
{"index": {"_index": self._index, "_type": "type1", "_id": "1"}},
{"name": "hiq", "age": 10},
{"index": {"_index": self._index, "_type": "type1", "_id": "2"}},
{"name": "hiq", "age": 10},
{"index": {"_index": self._index, "_type": "type1", "_id": "3"}},
{"name": "hiq", "age": 10}
]
@asyncio.coroutine
def go():
data = yield from self.cl.bulk(bulks)
self.assertFalse(data['errors'])
self.assertEqual(3, len(data['items']), data)
data = yield from self.cl.bulk(
bulks,
consistency='one',
refresh=True,
routing='hiq',
replication='async',
timeout='1s'
)
with self.assertRaises(TypeError):
yield from self.cl.bulk(bulks, consistency=1)
with self.assertRaises(ValueError):
yield from self.cl.bulk(bulks, consistency='1')
with self.assertRaises(TypeError):
yield from self.cl.bulk(bulks, replication=1)
with self.assertRaises(ValueError):
yield from self.cl.bulk(bulks, replication='1')
self.loop.run_until_complete(go())
def test_mget(self):
""" mget """
@asyncio.coroutine
def go():
yield from self.cl.index(
self._index, 'testdoc', MESSAGES[0], '1', refresh=True)
yield from self.cl.index(
self._index, 'testdoc', MESSAGES[1], '2', refresh=True)
yield from self.cl.index(
self._index, 'testdoc', MESSAGES[2], '3', refresh=True)
body = {
"docs": [
{"_index": self._index, "_type": "testdoc", "_id": "1"},
{"_index": self._index, "_type": "testdoc", "_id": "2"}
]
}
data = yield from self.cl.mget(body)
self.assertEqual(len(data['docs']), 2)
data = yield from self.cl.mget(
body,
_source_exclude='birthDate',
_source_include='user,skills',
_source=False,
fields='user,skills',
realtime=True,
refresh=True,
preference='random',
parent=''
)
self.assertIn('skills', data['docs'][0]['fields'], data)
self.assertIn('user', data['docs'][0]['fields'], data)
self.assertIn('skills', data['docs'][0]['_source'], data)
self.assertIn('user', data['docs'][0]['_source'], data)
yield from self.cl.mget(body, routing='Sidor')
self.loop.run_until_complete(go())
def test_suggest(self):
""" search """
@asyncio.coroutine
def go():
mapping = {
"testdoc": {
"properties": {
"birthDate": {
"type": "date",
"format": "dateOptionalTime"
},
"counter": {
"type": "long"
},
# this one is different
"message": {
"type": "completion"
},
"skills": {
"type": "string"
},
"user": {
"type": "string"
}
}
}
}
yield from self.cl.indices.create(self._index)
yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[1], '2',
refresh=True)
b = {
"my-suggestion": {
"text": "trying out",
"completion": {
"field": "message"
}
}
}
data = yield from self.cl.suggest(
self._index,
body=b,
)
results = data['my-suggestion'][0]['options']
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['text'], 'trying out Elasticsearch')
self.loop.run_until_complete(go())
def test_percolate(self):
@asyncio.coroutine
def go():
mapping = {
"testdoc": {
"properties": {
"message": {
"type": "string"
}
}
}
}
yield from self.cl.indices.create(self._index)
yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
percolator = {
"query": {
"match": {
"message": "bonsai tree"
}
}
}
# register percolator
yield from self.cl.index(self._index, '.percolator',
percolator, '1',
refresh=True)
b = {
"doc": {
"message": "A new bonsai tree in the office"
}
}
# percolate a doc from b
data = yield from self.cl.percolate(
self._index,
'testdoc',
body=b,
)
self.assertEqual(data['total'], 1)
self.assertEqual(
data['matches'][0],
{'_index': 'test_elasticsearch', '_id': '1'}
)
# percolate_count gives only count, no matches
data = yield from self.cl.count_percolate(
self._index,
'testdoc',
body=b,
)
self.assertEqual(data['total'], 1)
self.assertTrue('matches' not in data)
self.loop.run_until_complete(go())
def test_mpercolate(self):
@asyncio.coroutine
def go():
mapping = {
"testdoc": {
"properties": {
"message": {
"type": "string"
}
}
}
}
yield from self.cl.indices.create(self._index)
yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
percolator = {
"query": {
"match": {
"message": "bonsai tree"
}
}
}
# register percolator
yield from self.cl.index(self._index, '.percolator',
percolator, '1',
refresh=True)
body = [
{
'percolate': {
'index': self._index,
'type': 'testdoc',
}
},
{
"doc": {
"message": "A new bonsai tree in the office"
}
}
]
data = yield from self.cl.mpercolate(
body,
self._index,
'testdoc',
)
self.assertEqual(len(data['responses']), 1)
item = data['responses'][0]
self.assertEqual(item['total'], 1)
self.assertEqual(
item['matches'][0],
{'_index': 'test_elasticsearch', '_id': '1'}
)
self.loop.run_until_complete(go())
def test_termvector(self):
@asyncio.coroutine
def go():
mapping = {
"testdoc": {
"properties": {
"message": {
"type": "string",
"term_vector": "with_positions_offsets_payloads",
"store": True,
}
}
}
}
yield from self.cl.indices.create(self._index)
yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
doc = {
'message': 'Hello world',
}
yield from self.cl.index(self._index, 'testdoc',
doc, '1',
refresh=True)
data = yield from self.cl.termvector(self._index, 'testdoc', '1')
vector_data = data['term_vectors']['message']
self.assertEqual(vector_data['field_statistics'], {
"sum_doc_freq": 2,
"doc_count": 1,
"sum_ttf": 2
})
self.assertTrue('hello' in vector_data['terms'])
self.assertTrue('world' in vector_data['terms'])
self.loop.run_until_complete(go())
def test_mtermvectors(self):
@asyncio.coroutine
def go():
mapping = {
"testdoc": {
"properties": {
"message": {
"type": "string",
"term_vector": "with_positions_offsets_payloads",
"store": True,
}
}
}
}
yield from self.cl.indices.create(self._index)
yield from self.cl.indices.put_mapping(
self._index,
'testdoc',
mapping,
)
doc = {
'message': 'Hello world',
}
yield from self.cl.index(self._index, 'testdoc',
doc, '1',
refresh=True)
doc = {
'message': 'Second term',
}
yield from self.cl.index(self._index, 'testdoc',
doc, '2',
refresh=True)
data = yield from self.cl.mtermvectors(
self._index, 'testdoc', ids='1,2'
)
self.assertEqual(len(data['docs']), 2)
self.assertTrue('term_vectors' in data['docs'][0])
self.assertTrue('term_vectors' in data['docs'][1])
self.loop.run_until_complete(go())
def test_scripts_management(self):
@asyncio.coroutine
def go():
script = {'script': 'log(_score * 2)'}
# adding
yield from self.cl.put_script('groovy', 'test_script', script)
# getting and checking
got_script = yield from self.cl.get_script('groovy', 'test_script')
self.assertEqual(script['script'], got_script['script'])
# deleting
yield from self.cl.delete_script('groovy', 'test_script')
with self.assertRaises(NotFoundError):
got_script = yield from self.cl.get_script(
'groovy', 'test_script'
)
self.loop.run_until_complete(go())
def test_scripts_execution(self):
@asyncio.coroutine
def go():
script = {
'script': '2*val',
}
query = {
"query": {
"match": {
"user": "******"
}
},
"script_fields": {
"test1": {
"lang": "groovy",
"script_id": "calculate-score",
"params": {
"val": 2,
}
}
}
}
yield from self.cl.index(self._index, 'testdoc',
MESSAGES[0], '1',
refresh=True)
yield from self.cl.put_script('groovy', 'calculate-score', script)
data = yield from self.cl.search(self._index, 'testdoc', query)
res = data['hits']['hits'][0]['fields']['test1'][0]
self.assertEqual(res, 4) # 2*2
self.loop.run_until_complete(go())
def test_templates_management(self):
@asyncio.coroutine
def go():
template = {
"template": {
"query": {
"match": {
"user": "******"
}
}
}
}
yield from self.cl.put_template('test_template', template)
data = yield from self.cl.get_template('test_template')
self.assertTrue(template, data)
yield from self.cl.delete_template('test_template')
with self.assertRaises(NotFoundError):
yield from self.cl.get_template('test_template')
self.loop.run_until_complete(go())
def test_template_search(self):
@asyncio.coroutine
def go():
template = {
"template": {
"query": {
"match": {
"user": "******"
}
}
}
}
search_body = {
"template": {
"id": "test_template"
},
"params": {
"query_string": "Johny Mnemonic"
}
}
yield from self.cl.index(
self._index, 'testdoc', MESSAGES[0], '1',
refresh=True
)
yield from self.cl.put_template('test_template', template)
data = yield from self.cl.search_template(
self._index, 'testdoc', body=search_body
)
self.assertEqual(data['hits']['total'], 1)
self.loop.run_until_complete(go())
def test_search_shards(self):
@asyncio.coroutine
def go():
yield from self.cl.index(
self._index, 'testdoc', MESSAGES[0], '1',
refresh=True
)
data = yield from self.cl.search_shards(
self._index, 'testdoc'
)
self.assertTrue('nodes' in data)
self.assertTrue(len(data['nodes']) > 0)
self.assertTrue('shards' in data)
self.assertTrue(len(data['shards']) > 0)
self.loop.run_until_complete(go())
def test_mlt(self):
@asyncio.coroutine
def go():
msg = MESSAGES[0].copy()
# mlt needs quite a lot of text to work
msg['message'] = '''
Additionally, More Like This can find documents that are "like"
a set of chosen documents. The syntax to specify one or more
documents is similar to the Multi GET API, and supports the
ids or docs array. If only one document is specified,
the query behaves the same as the More Like This API.'''
# and quite a lot of documents
for i in range(50):
yield from self.cl.index(
self._index, 'testdoc', msg, str(i),
refresh=True
)
data = yield from self.cl.mlt(
self._index, 'testdoc', '1'
)
# initial document is not included
self.assertEqual(data['hits']['total'], 49)
self.assertEqual(data['hits']['hits'][0]['_source'], msg)
self.loop.run_until_complete(go())
def insert_emotions(project_seq, document_data):
some_bulks = ''
bulk_result = 0
try:
bica_ip, bica_port, concept_id = mariadbclient.get_bica_info(
project_seq)
bica_result = request2bica(
bica_ip, bica_port, concept_id,
document_data['_source']['doc_title'] + ' ' +
document_data['_source']['doc_content'])
if bica_result:
json_result = json.loads(bica_result)
from elasticsearch import Elasticsearch
es_client = Elasticsearch(":".join([es_ip, str(es_port)]))
try:
fts = [
make_bulk(document_data, r) for r in json_result['result']
]
#t.cancel()
some_bulks = yield from asyncio.gather(*fts)
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0, some_bulks)))[0]
except exceptions.ConnectionTimeout as timeoutError:
retry = 0
logger.error("[insert_emotions] %s (retry:%d)" %
(str(timeoutError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)))[0]
break
except exceptions.ConnectionTimeout as timeoutError:
logger.error("[insert_emotions] %s (retry:%d)" %
(str(timeoutError), retry))
continue
except aiohttp.client_exceptions.ClientConnectorError as connectError:
retry = 0
logger.error("[insert_emotions] %s (retry:%d)" %
(str(connectError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)))[0]
break
except aiohttp.client_exceptions.ClientConnectorError as connectError:
logger.error("[insert_emotions] %s (retry:%d)" %
(str(connectError), retry))
continue
except OSError as oserror:
retry = 0
logger.error("[insert_emotions] %s (retry:%d)" %
(str(oserror), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)))[0]
break
except OSError as oserror:
logger.error("[insert_emotions] %s (retry:%d)" %
(str(oserror), retry))
continue
except urllib3.exceptions.NewConnectionError as connectionError:
retry = 0
logger.error("[insert_emotions] %s (retry:%d)" %
(str(connectionError), retry))
while retry <= 5:
retry += 1
print("10초 간 쉬었다가 다시!\n")
time.sleep(10)
try:
print("색인 {0}번째 재시도..".format(retry))
bulk_result += helpers.bulk(
es_client,
list(filter(lambda x: x and len(x) > 0,
some_bulks)))[0]
break
except urllib3.exceptions.NewConnectionError as connectionError:
logger.error("[insert_emotions] %s (retry:%d)" %
(str(connectionError), retry))
continue
except:
ex = traceback.format_exc()
logger.error(
"[insert_emotions] unknown error. Traceback >> %s " % ex)
logger.debug("%d are successfully inserted." % bulk_result)
except:
ex = traceback.format_exc()
logger.error("[insert_emotions] unknown error. Traceback >> %s " % ex)
