class ESConnection(object):
def __init__(self, host='localhost', port='9200', io_loop=None):
self.io_loop = io_loop or IOLoop.instance()
self.url = "http://%(host)s:%(port)s" % {"host": host, "port": port}
self.bulk = BulkList()
self.client = AsyncHTTPClient(self.io_loop)
def create_path(self, method, **kwargs):
index = kwargs.get('index', '_all')
type_ = '/' + kwargs.get('type') if 'type' in kwargs else ''
size = kwargs.get('size', 10)
page = kwargs.get('page', 1)
from_ = (page - 1) * size
routing = kwargs.get('routing', '')
jsonp_callback = kwargs.get('jsonp_callback', '')
parameters = {'from': from_, 'size': size}
if routing:
parameters["routing"] = routing
path = "/%(index)s%(type)s/_%(method)s?%(querystring)s%(jsonp_callback)s" % {
"querystring": urlencode(parameters),
"method": method,
"index": index,
"type": type_,
"jsonp_callback": "&callback=" + jsonp_callback if jsonp_callback else ""
}
return path
def search(self, callback, **kwargs):
path = self.create_path("search", **kwargs)
source = json.dumps(kwargs.get('source', {"query": {"query_string": {"query": "*"}}}))
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
def apply_search(self, callback):
path = "/_msearch"
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(url, method="POST", body=source)
self.client.fetch(request=request_http, callback=callback)
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback)
def get(self, index, type, uid, callback):
def to_dict_callback(response):
source = json.loads(response.body).get('_source', {})
callback(source)
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, to_dict_callback)
class ESConnection(object):
def __init__(self, host='localhost', port='9200', io_loop=None, protocol='http'):
self.io_loop = io_loop or IOLoop.instance()
self.url = "%(protocol)s://%(host)s:%(port)s" % {"protocol": protocol, "host": host, "port": port}
self.bulk = BulkList()
self.client = AsyncHTTPClient(self.io_loop)
self.httprequest_kwargs = {} # extra kwargs passed to tornado's HTTPRequest class
# e.g. request_timeout
def create_path(self, method, **kwargs):
index = kwargs.get('index', '_all')
type_ = '/' + kwargs.get('type') if 'type' in kwargs else ''
parameters = {}
for param in ['size', 'from', 'routing', 'search_type']:
value = kwargs.get(param, None)
if value:
parameters[param] = value
if 'page' in kwargs:
parameters.setdefault('size', 10)
parameters['from'] = (kwargs['page'] - 1) * parameters['size']
jsonp_callback = kwargs.get('jsonp_callback', '')
if jsonp_callback:
parameters['callback'] = jsonp_callback
path = "/%(index)s%(type)s/_%(method)s" % {
"method": method,
"index": index,
"type": type_
}
if parameters:
path += '?' + urlencode(parameters)
return path
@return_future
def search(self, callback, **kwargs):
path = self.create_path("search", **kwargs)
source = json_encode(kwargs.get('source', {
"query": {
"match_all": {}
}
}))
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
@return_future
def apply_search(self, callback, params={}):
path = "/_msearch"
if params:
path = "%s?%s" % (path, urlencode(params))
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(url, method="POST", body=source, **self.httprequest_kwargs)
self.client.fetch(request=request_http, callback=callback)
@return_future
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback, **self.httprequest_kwargs)
@return_future
def get(self, index, type, uid, callback, parameters=None):
def to_dict_callback(response):
source = json_decode(response.body)
callback(source)
self.request_document(index, type, uid, callback=to_dict_callback, parameters=parameters)
@return_future
def put(self, index, type, uid, contents, parameters=None, callback=None):
self.request_document(
index, type, uid, "PUT", body=json_encode(contents),
parameters=parameters, callback=callback)
@return_future
def delete(self, index, type, uid, parameters=None, callback=None):
self.request_document(index, type, uid, "DELETE", parameters=parameters, callback=callback)
@return_future
def count(self, index="_all", type=None, source='', parameters=None, callback=None):
path = '/{}'.format(index)
if type:
path += '/{}'.format(type)
path += '/_count'
if parameters:
path += '?{}'.format(urlencode(parameters or {}))
if source:
source = json_encode(source)
self.post_by_path(path=path, callback=callback, source=source)
def request_document(self, index, type, uid, method="GET", body=None, parameters=None, callback=None):
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s?%(querystring)s' % {
"url": self.url,
"path": path,
"querystring": urlencode(parameters or {})
}
request_arguments = dict(self.httprequest_kwargs)
request_arguments['method'] = method
if body is not None:
request_arguments['body'] = body
request = HTTPRequest(url, **request_arguments)
self.client.fetch(request, callback)
class ESConnection(object):
_MATCH_ALL_QUERY = {"query": {"match_all": {}}}
def __init__(self,
host='localhost',
port='9200',
io_loop=None,
protocol='http',
custom_client=None,
http_request_kwargs=None):
self.io_loop = io_loop or IOLoop.instance()
self.url = "%(protocol)s://%(host)s:%(port)s" % {
"protocol": protocol,
"host": host,
"port": port
}
self.bulk = BulkList()
self.client = custom_client or AsyncHTTPClient(self.io_loop)
# extra kwargs passed to tornado's HTTPRequest class e.g. request_timeout
self.http_request_kwargs = http_request_kwargs or {}
@staticmethod
def _create_query_string(params):
"""
Support Elasticsearch 5.X
"""
parameters = params or {}
for param, value in parameters.items():
param_value = str(value).lower() if isinstance(value,
bool) else value
parameters[param] = param_value
return urlencode(parameters)
@classmethod
def from_uri(cls,
uri,
io_loop=None,
custom_client=None,
http_request_kwargs=None):
parsed = urlparse(uri)
if not parsed.hostname or not parsed.scheme:
raise ValueError('Invalid URI')
return cls(host=parsed.hostname,
protocol=parsed.scheme,
port=parsed.port,
io_loop=io_loop,
custom_client=custom_client,
http_request_kwargs=http_request_kwargs)
@staticmethod
def create_path(method, index='_all', type='', **kwargs):
query_string = ESConnection._create_query_string(kwargs)
path = "/%(index)s/%(type)s/_%(method)s" % {
"method": method,
"index": index,
"type": type
}
if query_string:
path += '?' + query_string
return path
@return_future
def search(self, callback, index='_all', type='', source=None, **kwargs):
source = json_encode(source or self._MATCH_ALL_QUERY)
path = self.create_path("search", index=index, type=type, **kwargs)
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
@return_future
def apply_search(self, callback, params=None):
params = params or {}
path = "/_msearch"
if params:
path = "%s?%s" % (path, urlencode(params))
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(
url,
method="POST",
body=source,
headers={'Content-Type': 'application/json'},
**self.http_request_kwargs)
self.client.fetch(request=request_http, callback=callback)
@return_future
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback, **self.http_request_kwargs)
@return_future
def get(self, index, type, uid, callback, parameters=None):
def to_dict_callback(response):
source = json_decode(response.body)
callback(source)
self.request_document(index,
type,
uid,
callback=to_dict_callback,
parameters=parameters)
@return_future
def put(self, index, type, uid, contents, parameters=None, callback=None):
self.request_document(index,
type,
uid,
"PUT",
body=json_encode(contents),
parameters=parameters,
callback=callback)
@return_future
def update(self, index, type, uid, contents, callback=None):
path = "/%(index)s/%(type)s/%(uid)s/_update" % {
"index": index,
"type": type,
"uid": uid
}
partial = {"doc": contents}
self.post_by_path(path, callback, source=json_encode(partial))
@return_future
def delete(self, index, type, uid, parameters=None, callback=None):
self.request_document(index,
type,
uid,
"DELETE",
parameters=parameters,
callback=callback)
@return_future
def count(self,
index="_all",
type='',
source='',
parameters=None,
callback=None):
"""
The query can either be provided using a simple query string as a parameter 'q',
or using the Query DSL defined within the request body (source).
Notice there are additional query string parameters that could be added only with the first option.
"""
parameters = parameters or {}
path = self.create_path('count', index=index, type=type, **parameters)
if source:
source = json_encode(source)
self.post_by_path(path=path, callback=callback, source=source)
def request_document(self,
index,
type,
uid,
method="GET",
body=None,
parameters=None,
callback=None):
query_string = ESConnection._create_query_string(parameters)
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s?%(querystring)s' % {
"url": self.url,
"path": path,
"querystring": query_string
}
request_arguments = dict(self.http_request_kwargs)
request_arguments['method'] = method
if body is not None:
request_arguments['body'] = body
request = HTTPRequest(url,
headers={'Content-Type': 'application/json'},
**request_arguments)
self.client.fetch(request, callback)
class ESConnection(object):
def __init__(self, host='localhost', port='9200', io_loop=None, protocol='http'):
self.io_loop = io_loop or IOLoop.instance()
self.url = "%(protocol)s://%(host)s:%(port)s" % {"protocol": protocol, "host": host, "port": port}
self.bulk = BulkList()
self.client = AsyncHTTPClient(self.io_loop)
self.httprequest_kwargs = {} # extra kwargs passed to tornado's HTTPRequest class e.g. request_timeout
def create_path(self, method, **kwargs):
index = kwargs.pop('index', '_all')
doc_type = '/%s' % kwargs.pop('type', '')
parameters = {}
for param, value in kwargs.items():
parameters[param] = value
path = "/%(index)s%(type)s/_%(method)s" % {
"method": method,
"index": index,
"type": doc_type
}
if parameters:
path += '?' + urlencode(parameters)
return path
@return_future
def search(self, callback, **kwargs):
path = self.create_path("search", **kwargs)
source = json_encode(kwargs.get('source', {
"query": {
"match_all": {}
}
}))
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
@return_future
def apply_search(self, callback, params={}):
path = "/_msearch"
if params:
path = "%s?%s" % (path, urlencode(params))
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(url, method="POST", body=source, **self.httprequest_kwargs)
self.client.fetch(request=request_http, callback=callback)
@return_future
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback, **self.httprequest_kwargs)
@return_future
def get(self, index, type, uid, callback, parameters=None):
def to_dict_callback(response):
source = json_decode(response.body)
callback(source)
self.request_document(index, type, uid, callback=to_dict_callback, parameters=parameters)
@return_future
def put(self, index, type, uid, contents, parameters=None, callback=None):
self.request_document(
index, type, uid, "PUT", body=json_encode(contents),
parameters=parameters, callback=callback)
@return_future
def delete(self, index, type, uid, parameters=None, callback=None):
self.request_document(index, type, uid, "DELETE", parameters=parameters, callback=callback)
@return_future
def count(self, index="_all", type=None, source='', parameters=None, callback=None):
path = '/{}'.format(index)
if type:
path += '/{}'.format(type)
path += '/_count'
if parameters:
path += '?{}'.format(urlencode(parameters or {}))
if source:
source = json_encode(source)
self.post_by_path(path=path, callback=callback, source=source)
def request_document(self, index, type, uid, method="GET", body=None, parameters=None, callback=None):
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s?%(querystring)s' % {
"url": self.url,
"path": path,
"querystring": urlencode(parameters or {})
}
request_arguments = dict(self.httprequest_kwargs)
request_arguments['method'] = method
if body is not None:
request_arguments['body'] = body
request = HTTPRequest(url, **request_arguments)
self.client.fetch(request, callback)
class ESConnection(object):
_MATCH_ALL_QUERY = {"query": {"match_all": {}}}
def __init__(self, host='localhost', port='9200', io_loop=None, protocol='http', custom_client=None,
http_request_kwargs=None):
self.io_loop = io_loop or IOLoop.instance()
self.url = "%(protocol)s://%(host)s:%(port)s" % {"protocol": protocol, "host": host, "port": port}
self.bulk = BulkList()
self.client = custom_client or AsyncHTTPClient(self.io_loop)
# extra kwargs passed to tornado's HTTPRequest class e.g. request_timeout
self.http_request_kwargs = http_request_kwargs or {}
@staticmethod
def _create_query_string(params):
"""
Support Elasticsearch 5.X
"""
parameters = params or {}
for param, value in parameters.items():
param_value = str(value).lower() if isinstance(value, bool) else value
parameters[param] = param_value
return urlencode(parameters)
@classmethod
def from_uri(cls, uri, io_loop=None, custom_client=None, http_request_kwargs=None):
parsed = urlparse(uri)
if not parsed.hostname or not parsed.scheme:
raise ValueError('Invalid URI')
return cls(host=parsed.hostname, protocol=parsed.scheme, port=parsed.port, io_loop=io_loop,
custom_client=custom_client, http_request_kwargs=http_request_kwargs)
@staticmethod
def create_path(method, index='_all', type='', **kwargs):
query_string = ESConnection._create_query_string(kwargs)
path = "/%(index)s/%(type)s/_%(method)s" % {"method": method, "index": index, "type": type}
if query_string:
path += '?' + query_string
return path
@return_future
def search(self, callback, index='_all', type='', source=None, **kwargs):
source = json_encode(source or self._MATCH_ALL_QUERY)
path = self.create_path("search", index=index, type=type, **kwargs)
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
@return_future
def apply_search(self, callback, params=None):
params = params or {}
path = "/_msearch"
if params:
path = "%s?%s" % (path, urlencode(params))
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(url, method="POST", body=source, **self.http_request_kwargs)
self.client.fetch(request=request_http, callback=callback)
@return_future
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback, **self.http_request_kwargs)
@return_future
def get(self, index, type, uid, callback, parameters=None):
def to_dict_callback(response):
source = json_decode(response.body)
callback(source)
self.request_document(index, type, uid, callback=to_dict_callback, parameters=parameters)
@return_future
def put(self, index, type, uid, contents, parameters=None, callback=None):
self.request_document(
index, type, uid, "PUT", body=json_encode(contents),
parameters=parameters, callback=callback)
@return_future
def update(self, index, type, uid, contents, callback=None):
path = "/%(index)s/%(type)s/%(uid)s/_update" % {
"index": index,
"type": type,
"uid": uid
}
partial = { "doc": contents }
self.post_by_path(path, callback, source=json_encode(partial))
@return_future
def delete(self, index, type, uid, parameters=None, callback=None):
self.request_document(index, type, uid, "DELETE", parameters=parameters, callback=callback)
@return_future
def count(self, index="_all", type='', source='', parameters=None, callback=None):
"""
The query can either be provided using a simple query string as a parameter 'q',
or using the Query DSL defined within the request body (source).
Notice there are additional query string parameters that could be added only with the first option.
"""
parameters = parameters or {}
path = self.create_path('count', index=index, type=type, **parameters)
if source:
source = json_encode(source)
self.post_by_path(path=path, callback=callback, source=source)
def request_document(self, index, type, uid, method="GET", body=None, parameters=None, callback=None):
query_string = ESConnection._create_query_string(parameters)
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s?%(querystring)s' % {
"url": self.url,
"path": path,
"querystring": query_string
}
request_arguments = dict(self.http_request_kwargs)
request_arguments['method'] = method
if body is not None:
request_arguments['body'] = body
request = HTTPRequest(url, **request_arguments)
self.client.fetch(request, callback)
class ESConnection(object):
def __init__(self,
host='localhost',
port='9200',
io_loop=None,
protocol='http',
custom_client=None):
self.io_loop = io_loop or IOLoop.instance()
self.url = "%(protocol)s://%(host)s:%(port)s" % {
"protocol": protocol,
"host": host,
"port": port
}
self.bulk = BulkList()
self.client = custom_client or AsyncHTTPClient(self.io_loop)
self.httprequest_kwargs = {
} # extra kwargs passed to tornado's HTTPRequest class e.g. request_timeout
@staticmethod
def from_uri(uri, io_loop=None, custom_client=None):
parsed = urlparse(uri)
if not parsed.hostname or not parsed.scheme:
raise ValueError('Invalid URI')
return ESConnection(
**{
'host': parsed.hostname,
'protocol': parsed.scheme,
'port': parsed.port,
'io_loop': io_loop,
'custom_client': custom_client
})
def create_path(self, method, **kwargs):
index = kwargs.pop('index', '_all')
doc_type = '/%s' % kwargs.pop('type', '')
parameters = {}
for param, value in kwargs.items():
parameters[param] = value
path = "/%(index)s%(type)s/_%(method)s" % {
"method": method,
"index": index,
"type": doc_type
}
if parameters:
path += '?' + urlencode(parameters)
return path
@return_future
def search(self, callback, **kwargs):
path = self.create_path("search", **kwargs)
source = json_encode(kwargs.get('source', {"query": {
"match_all": {}
}}))
self.post_by_path(path, callback, source)
def multi_search(self, index, source):
self.bulk.add(index, source)
@return_future
def apply_search(self, callback, params={}):
path = "/_msearch"
if params:
path = "%s?%s" % (path, urlencode(params))
source = self.bulk.prepare_search()
self.post_by_path(path, callback, source=source)
def post_by_path(self, path, callback, source):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
request_http = HTTPRequest(url,
method="POST",
body=source,
**self.httprequest_kwargs)
self.client.fetch(request=request_http, callback=callback)
@return_future
def get_by_path(self, path, callback):
url = '%(url)s%(path)s' % {"url": self.url, "path": path}
self.client.fetch(url, callback, **self.httprequest_kwargs)
@return_future
def get(self, index, type, uid, callback, parameters=None):
def to_dict_callback(response):
source = json_decode(response.body)
callback(source)
self.request_document(index,
type,
uid,
callback=to_dict_callback,
parameters=parameters)
@return_future
def put(self, index, type, uid, contents, parameters=None, callback=None):
self.request_document(index,
type,
uid,
"PUT",
body=json_encode(contents),
parameters=parameters,
callback=callback)
@return_future
def update(self, index, type, uid, contents, callback=None):
path = "/%(index)s/%(type)s/%(uid)s/_update" % {
"index": index,
"type": type,
"uid": uid
}
partial = {"doc": contents}
self.post_by_path(path, callback, source=json_encode(partial))
@return_future
def delete(self, index, type, uid, parameters=None, callback=None):
self.request_document(index,
type,
uid,
"DELETE",
parameters=parameters,
callback=callback)
@return_future
def count(self,
index="_all",
type=None,
source='',
parameters=None,
callback=None):
path = '/{}'.format(index)
if type:
path += '/{}'.format(type)
path += '/_count'
if parameters:
path += '?{}'.format(urlencode(parameters or {}))
if source:
source = json_encode(source)
self.post_by_path(path=path, callback=callback, source=source)
def request_document(self,
index,
type,
uid,
method="GET",
body=None,
parameters=None,
callback=None):
path = '/{index}/{type}/{uid}'.format(**locals())
url = '%(url)s%(path)s?%(querystring)s' % {
"url": self.url,
"path": path,
"querystring": urlencode(parameters or {})
}
request_arguments = dict(self.httprequest_kwargs)
request_arguments['method'] = method
if body is not None:
request_arguments['body'] = body
request = HTTPRequest(url, **request_arguments)
self.client.fetch(request, callback)