def bulk_post(data_list,
host=METADATA_ES_SERVER,
port=ES_PORT,
timeout=es_utils.DEFAULT_TIMEOUT,
index=INDEX_METADATA,
retries=DEFAULT_BULK_POST_RETRIES,
*args,
**kwargs):
"""This function takes a series of arguments which are passed to the
es_utils.ESMetadata constructor, and a list of metadata, then upload to
Elasticsearch server using Elasticsearch bulk API. This can greatly nhance
the performance of uploading data using HTTP.
For an explanation of each argument, see those functions in es_utils.
"""
if not host:
return True
raise Exception(str(host) + '+' * 99)
esmd = es_utils.ESMetadata(use_http=True,
host=host,
port=port,
timeout=timeout,
index=index,
udp_port=ES_UDP_PORT)
# bulk post may fail due to the amount of data, retry several times.
for _ in range(retries):
if esmd.bulk_post(data_list, *args, **kwargs):
return True
return False
def query(host=METADATA_ES_SERVER,
port=ES_PORT,
timeout=es_utils.DEFAULT_TIMEOUT,
index=INDEX_METADATA,
*args,
**kwargs):
"""This function takes a series of arguments which are passed to the
es_utils.ESMetadata constructor, and any other arguments are passed to
its query() function. For an explanation of each, see those functions in
es_utils.
"""
esmd = es_utils.ESMetadata(use_http=True,
host=host,
port=port,
timeout=timeout,
index=index,
udp_port=0)
return esmd.query(*args, **kwargs)
def post(use_http=ES_USE_HTTP,
host=METADATA_ES_SERVER,
port=ES_PORT,
timeout=es_utils.DEFAULT_TIMEOUT,
index=INDEX_METADATA,
udp_port=ES_UDP_PORT,
*args,
**kwargs):
"""This function takes a series of arguments which are passed to the
es_utils.ESMetadata constructor, and any other arguments are passed to
its post() function. For an explanation of each, see those functions in
es_utils.
"""
if not host:
return
esmd = es_utils.ESMetadata(use_http=use_http,
host=host,
port=port,
timeout=timeout,
index=index,
udp_port=udp_port)
return esmd.post(*args, **kwargs)