def set_refresh_interval(self, seconds):
if seconds <= 0:
interval = -1
else:
interval = unicode(seconds) + "s"
if self.cluster.version.startswith("0.90."):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'
)
result = convert.json2value(utf82unicode(response.all_content))
if not result.ok:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}')
)
result = convert.json2value(utf82unicode(response.all_content))
if not result.acknowledged:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
else:
Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
def set_refresh_interval(self, seconds):
if seconds <= 0:
interval = -1
else:
interval = unicode(seconds) + "s"
if self.cluster.version.startswith("0.90."):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'
)
result = convert.json2value(utf82unicode(response.all_content))
if not result.ok:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}')
)
result = convert.json2value(utf82unicode(response.all_content))
if not result.acknowledged:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
else:
Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = convert.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = convert.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def _value2json(value, _buffer):
try:
_class = value.__class__
if value is None:
append(_buffer, u"null")
return
elif value is True:
append(_buffer, u"true")
return
elif value is False:
append(_buffer, u"false")
return
type = value.__class__
if type is str:
append(_buffer, u"\"")
try:
v = utf82unicode(value)
except Exception, e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\"")
elif type is unicode:
append(_buffer, u"\"")
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\"")
def _value2json(value, _buffer):
try:
if value is None:
append(_buffer, "null")
return
elif value is True:
append(_buffer, "true")
return
elif value is False:
append(_buffer, "false")
return
elif isinstance(value, Mapping):
if value:
_dict2json(value, _buffer)
else:
append(_buffer, "{}")
return
type = value.__class__
if type is str:
append(_buffer, '"')
try:
v = utf82unicode(value)
except Exception, e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, '"')
elif type is unicode:
append(_buffer, '"')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, '"')
def _typed_encode(value, _buffer):
try:
if value is None:
append(_buffer, u'{"$value": null}')
return
elif value is True:
append(_buffer, u'{"$value": true}')
return
elif value is False:
append(_buffer, u'{"$value": false}')
return
_type = value.__class__
if _type in (dict, Dict):
if value:
_dict2json(value, _buffer)
else:
append(_buffer, u'{"$object": "."}')
elif _type is str:
append(_buffer, u'{"$value": "')
try:
v = utf82unicode(value)
except Exception, e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type is unicode:
append(_buffer, u'{"$value": "')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
def pretty_json(value):
try:
if scrub(value) is None:
return "null"
elif isinstance(value, basestring):
if isinstance(value, str):
value = utf82unicode(value)
try:
return quote(value)
except Exception, e:
from pyLibrary.debugs.logs import Log
try:
Log.note(
"try explicit convert of string with length {{length}}",
length=len(value))
acc = [u"\""]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception, h:
c2 = c
c3 = unicode(c2)
acc.append(c3)
except BaseException, g:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(u"\"")
output = u"".join(acc)
Log.note("return value of length {{length}}",
length=len(output))
return output
def delete_index(self, index_name):
if not isinstance(index_name, unicode):
Log.error("expecting an index name")
if self.debug:
Log.note("Deleting index {{index}}", index=index_name)
# REMOVE ALL ALIASES TOO
aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None]
if aliases:
self.post(
path="/_aliases",
data={"actions": [{"remove": a} for a in aliases]}
)
url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name
try:
response = http.delete(url)
if response.status_code != 200:
Log.error("Expecting a 200, got {{code}}", code=response.status_code)
details = convert.json2value(utf82unicode(response.content))
if self.debug:
Log.note("delete response {{response}}", response=details)
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def delete_index(self, index_name):
if not isinstance(index_name, unicode):
Log.error("expecting an index name")
if self.debug:
Log.note("Deleting index {{index}}", index=index_name)
# REMOVE ALL ALIASES TOO
aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None]
if aliases:
self.post(
path="/_aliases",
data={"actions": [{"remove": a} for a in aliases]}
)
url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name
try:
response = http.delete(url)
if response.status_code != 200:
Log.error("Expecting a 200")
details = convert.json2value(utf82unicode(response.content))
if self.debug:
Log.note("delete response {{response}}", response=details)
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def pretty_json(value):
try:
if scrub(value) is None:
return "null"
elif isinstance(value, basestring):
if isinstance(value, str):
value = utf82unicode(value)
try:
return quote(value)
except Exception, e:
from pyLibrary.debugs.logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length= len(value))
acc = [u"\""]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception, h:
c2 = c
c3 = unicode(c2)
acc.append(c3)
except BaseException, g:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(u"\"")
output = u"".join(acc)
Log.note("return value of length {{length}}", length= len(output))
return output
def head(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.head(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.content)[:130])
if response.content:
details = wrap(convert.json2value(utf82unicode(response.content)))
if details.error:
Log.error(details.error)
return details
else:
return None # WE DO NOT EXPECT content WITH HEAD REQUEST
except Exception, e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def delete(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = convert.json2value(utf82unicode(http.delete(url, **kwargs).content))
if self.debug:
Log.note("delete response {{response}}", response= response)
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def _post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
if "data" in kwargs and not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get("data", "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason + ": " + response.all_content)
if self.debug:
Log.note("response: {{response}}",
response=utf82unicode(response.all_content)[:130])
details = convert.json2value(utf82unicode(response.all_content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(
r.replace(";", ";\n")
for r in details._shards.failures.reason))
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error("Problem with call to {{url}}" + suggestion +
"\n{{body|left(10000)}}",
url=url,
body=kwargs["data"][0:10000]
if self.debug else kwargs["data"][0:100],
cause=e)
else:
Log.error("Problem with call to {{url}}" + suggestion,
{"url": url}, e)
def delete(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = convert.json2value(
utf82unicode(http.delete(url, **kwargs).content))
if self.debug:
Log.note("delete response {{response}}", response=response)
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def head(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.head(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason + ": " + response.all_content)
if self.debug:
Log.note("response: {{response}}",
response=utf82unicode(response.all_content)[:130])
if response.all_content:
details = wrap(
convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
else:
return None # WE DO NOT EXPECT content WITH HEAD REQUEST
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def _dict2json(value, _buffer):
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
def _dict2json(value, _buffer):
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
def _post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
if "data" in kwargs and not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get("data", "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url= url, data= sample)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason+": "+response.content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.content)[:130])
details = convert.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures= "---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error("Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url= url,
body= kwargs["data"][0:10000] if self.debug else kwargs["data"][0:100], cause=e)
else:
Log.error("Problem with call to {{url}}" + suggestion, {"url": url}, e)
def output():
try:
path = self._filename
if path.startswith("~"):
home_path = os.path.expanduser("~")
path = home_path + path[1::]
with io.open(path, "rb") as f:
for line in f:
yield utf82unicode(line)
except Exception, e:
from pyLibrary.debugs.logs import Log
Log.error("Can not read line from {{filename}}", filename= self._filename, cause=e)
def _dict2json(value, _buffer):
prefix = u'{"$object": ".", "'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
if not isinstance(k, unicode):
Log.error("Expecting property name to be a string")
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_typed_encode(v, _buffer)
append(_buffer, u"}")
def put(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
if self.debug:
sample = kwargs["data"][:300]
Log.note("PUT {{url}}:\n{{data|indent}}", url= url, data= sample)
try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:])
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def put(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
if self.debug:
sample = kwargs["data"][:300]
Log.note("PUT {{url}}:\n{{data|indent}}", url= url, data= sample)
try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:])
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def output():
try:
path = self._filename
if path.startswith("~"):
home_path = os.path.expanduser("~")
path = home_path + path[1::]
with io.open(path, "rb") as f:
for line in f:
yield utf82unicode(line)
except Exception, e:
from pyLibrary.debugs.logs import Log
Log.error("Can not read line from {{filename}}",
filename=self._filename,
cause=e)
def _dict2json(value, _buffer):
try:
prefix = '{"'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = ', "'
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, '": ')
_value2json(v, _buffer)
append(_buffer, "}")
except Exception, e:
from pyLibrary.debugs.logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception, e:
from pyLibrary.debugs.logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
items = sorted(items, lambda a, b: value_compare(a[0], b[0]))
values = [unicode_key(k) + ": " + indent(pretty_json(v)).strip() for k, v in items if v != None]
return "{\n" + INDENT + (",\n" + INDENT).join(values) + "\n}"
except Exception, e:
from pyLibrary.debugs.logs import Log
from pyLibrary.collections import OR
if OR(not isinstance(k, basestring) for k in value.keys()):
Log.error("JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e)
Log.error("problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e)
elif value in (None, Null):
return "null"
elif isinstance(value, basestring):
if isinstance(value, str):
value = utf82unicode(value)
try:
return quote(value)
except Exception, e:
from pyLibrary.debugs.logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = ['"']
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception, h:
c2 = c
c3 = unicode(c2)
Log.error(
"JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
Log.error(
"problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
elif value in (None, Null):
return "null"
elif isinstance(value, basestring):
if isinstance(value, str):
value = utf82unicode(value)
try:
return quote(value)
except Exception, e:
from pyLibrary.debugs.logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = [u"\""]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = unicode(c2)