def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = convert.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = convert.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def json2value(json_string, params={}, flexible=False, leaves=False):
"""
:param json_string: THE JSON
:param params: STANDARD JSON PARAMS
:param flexible: REMOVE COMMENTS
:param leaves: ASSUME JSON KEYS ARE DOT-DELIMITED
:return: Python value
"""
if isinstance(json_string, str):
Log.error("only unicode json accepted")
try:
if flexible:
# REMOVE """COMMENTS""", # COMMENTS, //COMMENTS, AND \n \r
# DERIVED FROM https://github.com/jeads/datasource/blob/master/datasource/bases/BaseHub.py# L58
json_string = re.sub(r"\"\"\".*?\"\"\"", r"\n", json_string, flags=re.MULTILINE)
json_string = "\n".join(remove_line_comment(l) for l in json_string.split("\n"))
# ALLOW DICTIONARY'S NAME:VALUE LIST TO END WITH COMMA
json_string = re.sub(r",\s*\}", r"}", json_string)
# ALLOW LISTS TO END WITH COMMA
json_string = re.sub(r",\s*\]", r"]", json_string)
if params:
json_string = expand_template(json_string, params)
# LOOKUP REFERENCES
value = wrap(json_decoder(json_string))
if leaves:
value = wrap_leaves(value)
return value
except Exception, e:
e = Except.wrap(e)
if "Expecting '" in e and "' delimiter: line" in e:
line_index = int(strings.between(e.message, " line ", " column ")) - 1
column = int(strings.between(e.message, " column ", " ")) - 1
line = json_string.split("\n")[line_index].replace("\t", " ")
if column > 20:
sample = "..." + line[column - 20:]
pointer = " " + (" " * 20) + "^"
else:
sample = line
pointer = (" " * column) + "^"
if len(sample) > 43:
sample = sample[:43] + "..."
Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n")
base_str = unicode2utf8(strings.limit(json_string, 1000))
hexx_str = bytes2hex(base_str, " ")
try:
char_str = " " + (" ".join(c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str)
except Exception:
char_str = " "
Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e)
def write(self, template, params):
if params.get("template"):
# DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE
self.queue.add({"value": params})
else:
template = strings.limit(template, 2000)
self.queue.add({"value": {"template": template, "params": params}}, timeout=3*MINUTE)
return self
def write(self, template, params):
if params.get("template"):
# DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE
self.queue.add({"value": params})
else:
template = strings.limit(template, 2000)
self.queue.add({"value": {"template": template, "params": params}}, timeout=3 * MINUTE)
return self
def _deep_json_to_string(value, depth):
"""
:param value: SOME STRUCTURE
:param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
:return: FLATTER STRUCTURE
"""
if isinstance(value, Mapping):
if depth == 0:
return strings.limit(convert.value2json(value), LOG_STRING_LENGTH)
return {k: _deep_json_to_string(v, depth - 1) for k, v in value.items()}
elif isinstance(value, list):
return strings.limit(convert.value2json(value), LOG_STRING_LENGTH)
elif isinstance(value, (float, int, long)):
return value
elif isinstance(value, basestring):
return strings.limit(value, LOG_STRING_LENGTH)
else:
return strings.limit(convert.value2json(value), LOG_STRING_LENGTH)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def get(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.get(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(convert.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
line_index = int(strings.between(c.message, " line ", " column ")) - 1
column = int(strings.between(c.message, " column ", " ")) - 1
line = json_string.split("\n")[line_index].replace("\t", " ")
if column > 20:
sample = "..." + line[column - 20:]
pointer = " " + (" " * 20) + "^"
else:
sample = line
pointer = (" " * column) + "^"
if len(sample) > 43:
sample = sample[:43] + "..."
Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n")
base_str = unicode2utf8(strings.limit(json_string, 1000))
hexx_str = bytes2hex(base_str, " ")
try:
char_str = " " + " ".join((c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str)
except Exception, e:
char_str = " "
Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e)
def string2datetime(value, format=None):
return unix2datetime(Date(value, format).unix)
def str2datetime(value, format=None):
return unix2datetime(Date(value, format).unix)
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
if self.settings.read_only:
Log.error("Index opened in read only mode, no changes allowed")
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = random_id()
if "json" in r:
json_bytes = r["json"].encode("utf8")
elif "value" in r:
json_bytes = convert.value2json(r["value"]).encode("utf8")
else:
json_bytes = None
Log.error("Expecting every record given to have \"value\" or \"json\" property")
lines.append(b'{"index":{"_id": ' + convert.value2json(id).encode("utf8") + b'}}')
if self.settings.tjson:
lines.append(json2typed(json_bytes.decode('utf8')).encode('utf8'))
else:
lines.append(json_bytes)
del records
if not lines:
return
try:
data_bytes = b"\n".join(l for l in lines) + b"\n"
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e)
response = self.cluster.post(
self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout,
retry=self.settings.retry
)
items = response["items"]
fails = []
if self.cluster.version.startswith("0.90."):
for i, item in enumerate(items):
if not item.index.ok:
fails.append(i)
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
for i, item in enumerate(items):
if item.index.status not in [200, 201]:
fails.append(i)
else:
Log.error("version not supported {{version}}", version=self.cluster.version)
if fails:
Log.error("Problems with insert", cause=[
Except(
template="{{status}} {{error}} (and {{some}} others) while loading line id={{id}} into index {{index|quote}}:\n{{line}}",
status=items[i].index.status,
error=items[i].index.error,
some=len(fails) - 1,
line=strings.limit(lines[fails[0] * 2 + 1], 500 if not self.debug else 100000),
index=self.settings.index,
id=items[i].index._id
)
for i in fails
])
if self.debug:
Log.note("{{num}} documents added", num=len(items))
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
if self.settings.read_only:
Log.error("Index opened in read only mode, no changes allowed")
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = random_id()
if "json" in r:
json_bytes = r["json"].encode("utf8")
elif "value" in r:
json_bytes = convert.value2json(r["value"]).encode("utf8")
else:
json_bytes = None
Log.error("Expecting every record given to have \"value\" or \"json\" property")
lines.append(b'{"index":{"_id": ' + convert.value2json(id).encode("utf8") + b'}}')
if self.settings.tjson:
lines.append(json2typed(json_bytes.decode('utf8')).encode('utf8'))
else:
lines.append(json_bytes)
del records
if not lines:
return
with Timer("Add {{num}} documents to {{index}}", {"num": len(lines) / 2, "index":self.settings.index}, debug=self.debug):
try:
data_bytes = b"\n".join(l for l in lines) + b"\n"
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e)
response = self.cluster.post(
self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout,
retry=self.settings.retry
)
items = response["items"]
fails = []
if self.cluster.version.startswith("0.90."):
for i, item in enumerate(items):
if not item.index.ok:
fails.append(i)
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
for i, item in enumerate(items):
if item.index.status not in [200, 201]:
fails.append(i)
else:
Log.error("version not supported {{version}}", version=self.cluster.version)
if fails:
Log.error("Problems with insert", cause=[
Except(
template="{{status}} {{error}} (and {{some}} others) while loading line id={{id}} into index {{index|quote}}:\n{{line}}",
status=items[i].index.status,
error=items[i].index.error,
some=len(fails) - 1,
line=strings.limit(lines[fails[0] * 2 + 1], 500 if not self.debug else 100000),
index=self.settings.index,
id=items[i].index._id
)
for i in fails
])
except Exception, e:
if e.message.startswith("sequence item "):
Log.error("problem with {{data}}", data=repr(lines[int(e.message[14:16].strip())]), cause=e)
Log.error("problem sending to ES", e)
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
if self.settings.read_only:
Log.error("Index opened in read only mode, no changes allowed")
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = random_id()
if "json" in r:
# if id != coalesce(wrap(convert.json2value(r["json"])).value._id, id):
# Log.error("expecting _id to match")
json = r["json"]
elif "value" in r:
# if id != coalesce(wrap(r).value._id, id):
# Log.error("expecting _id to match")
json = convert.value2json(r["value"])
else:
json = None
Log.error("Expecting every record given to have \"value\" or \"json\" property")
lines.append('{"index":{"_id": ' + convert.value2json(id) + '}}')
if self.settings.tjson:
lines.append(json2typed(json))
else:
lines.append(json)
del records
if not lines:
return
try:
data_bytes = "\n".join(lines) + "\n"
data_bytes = data_bytes.encode("utf8")
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e)
response = self.cluster.post(
self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout
)
items = response["items"]
for i, item in enumerate(items):
if self.cluster.version.startswith("0.90."):
if not item.index.ok:
Log.error(
"{{error}} while loading line:\n{{line}}",
error=item.index.error,
line=lines[i * 2 + 1]
)
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
if item.index.status not in [200, 201]:
Log.error(
"{{num}} {{error}} while loading line id={{id}} into index {{index|quote}}:\n{{line}}",
num=item.index.status,
error=item.index.error,
line=strings.limit(lines[i * 2 + 1], 300),
index=self.settings.index,
id=item.index._id
)
else:
Log.error("version not supported {{version}}", version=self.cluster.version)
if self.debug:
Log.note("{{num}} documents added", num=len(items))
