def get_document_json_by_key(self,
resource_key: str,
resource_type: Resource) -> Any:
key_query = {
resource_type: Q(TERM_QUERY, key=resource_key),
}
response: Response = self.execute_queries(queries=key_query,
page_index=0,
results_per_page=1)
if len(response) < 1:
msg = f'No response from ES for key query {key_query[resource_type]}'
LOGGER.error(msg)
raise ElasticsearchException(msg)
response = response[0]
if response.success():
results_count = response.hits.total.value
if results_count == 1:
es_result = response.hits.hits[0]
return es_result
if results_count > 1:
msg = f'Key {key_query[resource_type]} is not unique to a single ES resource'
LOGGER.error(msg)
raise ValueError(msg)
else:
# no doc exists with given key in ES
msg = f"Requested key {resource_key} query returned no results in ES"
LOGGER.error(msg)
raise ValueError(msg)
else:
msg = f'Request to Elasticsearch failed: {response.failures}'
LOGGER.error(msg)
raise InternalServerError(msg)
def check_elasticsearch_index(self, recreate, settings, indexmappings):
if self.db == 'Elasticsearch':
if len(self.index) > 0:
e = self.es.indices.exists(index=self.index)
if e and recreate:
print("Deleting existing index " + self.index)
r = self.es.indices.delete(index=self.index,
params={"timeout": "20s"})
print(r)
# TODO: check delete-request return status
e = False
if not e:
if settings is None:
settings = {
"index.number_of_replicas": 0,
"index.write.wait_for_active_shards": 1,
"index.refresh_interval": "30s"
}
if indexmappings is None:
indexmappings = {}
r = self.es.indices.create(index=self.index,
params={"timeout": "10s"},
ignore=400,
body={
"settings": settings,
"mappings": indexmappings
})
if 'error' in r:
logger.error(r['error']['reason'])
raise ElasticsearchException(r['error']['reason'])
def test_get_website_document_with_error(self, mock_elastich):
mock_elastich.side_effect = mock.Mock(
side_effect=ElasticsearchException()) # noqa
with self.assertRaises(ElasticsearchException):
webiste_nosql = WebsiteNoSql()
webiste_nosql.get(1)
self.assertEqual(mock_elastich.call_count, 1)
def test_update_website_document_with_error(self, mock_elastich):
mock_elastich.side_effect = mock.Mock(
side_effect=ElasticsearchException()) # noqa
with self.assertRaises(ElasticsearchException):
body = {'urls': ['http://vtrmantovani.com.br']}
webiste_nosql = WebsiteNoSql()
webiste_nosql.update(1, body)
self.assertEqual(mock_elastich.call_count, 1)
def test_get_website_with_elasticsearch_exception(self, mock_get):
mock_get.side_effect = mock.Mock(side_effect=ElasticsearchException())
response = self.client.get(
'/api/website/1',
headers={'Authorization': 'PXaLogin apikey="xpo"'})
r = json.loads(response.data.decode('utf-8'))
self.assertEqual(response.status_code, 400)
self.assertEqual(r['error']['message'], "Some problems in bd")
def delete(self, id_webiste):
try:
return es.delete(index="website",
doc_type='website-type',
id=id_webiste)
except ElasticsearchException as e:
logger.exception(
"WebsiteNoSql delete. website_id={0}. Erro:{1}".format(
id_webiste, str(e))) # noqa
raise ElasticsearchException()
def create(self, id_webiste, body):
try:
es.index(index="website",
doc_type='website-type',
id=id_webiste,
body=body)
except ElasticsearchException as e:
logger.exception(
"WebsiteNoSql create. website_id={0}, body={1}, Erro:{1}".
format(id_webiste, str(body), str(e))) # noqa
raise ElasticsearchException()
def get(self, id_webiste):
try:
return es.get(index="website",
doc_type='website-type',
id=id_webiste)
except NotFoundError as e:
return None
except ElasticsearchException as e:
logger.exception(
"WebsiteNoSql get. website_id={0}. Erro:{1}".format(
id_webiste, str(e))) # noqa
raise ElasticsearchException()
def create(self, index, body=None, params=None):
if not self.es_client.indices.exists(index=index):
logger.info('index "' + index +
'": not exists! create now...\nnew mapping:')
if body and body.get('mappings') is not None:
logger.info(json_util.dumps(body.get('mappings')))
res = self.es_client.indices.create(index=index,
body=body,
params=params)
if res and res['acknowledged']:
return 'success'
else:
raise ElasticsearchException(res)
def create(index, mappings=None, settings=None):
if not es_client.indices.exists(index=index):
logger.info('index "' + index + '": not exists! create now...\nnew mapping:')
logger.info(json_util.dumps(mappings))
res = es_client.indices.create(index=index,
body={
'mappings': mappings,
'settings': settings
})
if res and res['acknowledged']:
return 'success'
else:
raise ElasticsearchException(res)
def test_search_website_with_elasticsearch_exception(self, mock_get):
mock_get.side_effect = mock.Mock(side_effect=ElasticsearchException())
website = Website()
website.url = "http://vtrmantovani.com.br"
website.status = Website.Status.DONE
db.session.add(website)
db.session.commit()
self.assertEqual(Website.query.count(), 1)
params = {"status": "DONE", "limit": 2, "offset": 0}
response = self.client.post("/api/search",
headers={
'Authorization': self.api_key,
'Content-Type': 'application/json'
},
data=json.dumps(params))
r = json.loads(response.data.decode('utf-8'))
self.assertEqual(response.status_code, 400)
self.assertEqual(r['error']['message'], "Some problems in bd")
def add_documents_to_index(
index_name: str,
document_cls: Type[BaseDocument],
document_dicts: Iterator[Mapping[str, object]],
*,
max_retries: int = 5,
num_procs: Optional[int] = None,
) -> None:
ensure_index_exists(index_name)
_LOGGER.debug("Indexing documents to index '{}'.", index_name)
def make_upsert_ops() -> Iterator[Mapping[str, object]]:
with Pool(processes=num_procs) as pool:
yield from pool.imap_unordered(
partial(
_make_upsert_op, index_name=index_name, document_cls=document_cls
),
document_dicts,
)
num_indexed = 0
for ok, result in streaming_bulk(
connections.get_connection(),
make_upsert_ops(),
max_retries=max_retries,
raise_on_error=False,
):
if ok:
num_indexed += 1
else:
_LOGGER.debug(
"Indexed {} documents before the following error.", num_indexed
)
raise ElasticsearchException(
f"An error occurred when indexing documents: {result}"
)
_LOGGER.debug("Successfully indexed {} documents.", num_indexed)
# try:
# # res = es.search(index="test_22", q="name:host-*", analyze_wildcard=True, sort="age:asc,height:desc",from_=4, size=2)
# # res = es.search(index="test", q="user-name:'zhangsan'", analyze_wildcard=True,from_=0, size=20)
# s = Search(using=es, index="test").query("match", user_name="zhangsan")
# res = s.execute()
# except Exception as exc:
# print(exc)
# exit(1)
# print(res)
# for i in res:
# print(i)
#
# for item in res["hits"]["hits"]:
# print(item["_source"]["user-name"])
try:
raise ElasticsearchException()
except Exception as exc:
print(type(exc))
# res = es.index(index="test_22..", doc_type="data", op_type="create", body={})
try:
res = indices_client.create(index="test-123")
except Exception as exc:
print("exc")
print(str(exc))
exit(1)
else:
print("else")
# res = indices_client.delete(index="server-logic-1")
print(res)
def bulk_error_2_elasticsearch_exception(items):
return ElasticsearchException(bulk_error_process(items))
def send_records(self, all_records, sub_api_name):
""" Sends records with API info to Elasticsearch
Args:
records (list of dicts): records from the API stored as a list of dicts
sub_api_name (string): name of the sub_api
"""
es_endpoints = {
es: Elasticsearch(host=es, port=self.es_port, timeout=90)
for es in self.es_hostnames
}
chunksize = self.chunksize
idx = self.index
if type(all_records) == list:
all_records = iter(all_records)
while True:
latest_ts = datetime.min.replace(tzinfo=dateutil.tz.tzutc())
records = list(itertools.islice(all_records, chunksize))
if not records:
break
# Map index names to records by timestamp
index_to_records = {}
start_idx = dateutil.parser.parse(
records[0]['event_time']).strftime(idx)
end_idx = dateutil.parser.parse(
records[-1]['event_time']).strftime(idx)
if self.in_order and start_idx == end_idx:
index_to_records[start_idx] = records
else:
# Map records to indices
for record in records:
ts = dateutil.parser.parse(record['event_time'])
latest_ts = max(latest_ts, ts)
index_to_records.setdefault(ts.strftime(idx),
[]).append(record)
_type = self.APP_NAME
for index, chunk in index_to_records.items():
self.apply_common_filters(chunk)
self.apply_specific_filters(chunk)
bulk = []
for doc in chunk:
doc_source = doc.get('{}_data'.format(self.APP_NAME), {})
_id = doc_source.get('_id')
if _id:
bulk.append(
simplejson.dumps({
'create': {
'_type': _type,
'_index': index,
'_id': _id
}
}))
doc_source.pop('_id')
else:
bulk.append(
simplejson.dumps(
{'create': {
'_type': _type,
'_index': index
}}))
bulk.append(simplejson.dumps(doc))
bulk = '\n'.join(bulk) + '\n'
for hostname, es in es_endpoints.iteritems():
for retries in range(5):
try:
es.bulk(body=bulk, index=index)
break
except ElasticsearchException as e:
self.log.exception(
'Attempt {0} for sending {1} to Elasticsearch failed.'
.format(retries, self.APP_NAME))
time.sleep(2**retries)
else:
raise ElasticsearchException(
'Upload from the {0} {1} API to {2} failed. '
'Original exception: {3}'.format(
self.APP_NAME, sub_api_name, hostname, e))
latest_ts = latest_ts.isoformat()
self.mark_as_completed(latest_ts, sub_api_name)
self.log.warn(
"{0} ES document(s) sent from the {1} {2} API for the {3} account"
.format(len(records), self.APP_NAME, sub_api_name,
self.domain))
logging.info('messages_sent_to_es', sub_api_name, len(records))