def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = Random.hex(40)
if "json" in r:
json = r["json"]
elif "value" in r:
json = convert.value2json(r["value"])
else:
json = None
Log.error("Expecting every record given to have \"value\" or \"json\" property")
lines.append('{"index":{"_id": ' + convert.value2json(id) + '}}')
lines.append(json)
del records
if not lines:
return
try:
data_bytes = "\n".join(lines) + "\n"
data_bytes = data_bytes.encode("utf8")
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}", lines= lines, cause=e)
response = self.cluster._post(
self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout
)
items = response["items"]
for i, item in enumerate(items):
if self.cluster.version.startswith("0.90."):
if not item.index.ok:
Log.error("{{error}} while loading line:\n{{line}}",
error= item.index.error,
line= lines[i * 2 + 1])
elif self.cluster.version.startswith("1.4."):
if item.index.status not in [200, 201]:
Log.error("{{error}} while loading line:\n{{line}}",
error= item.index.error,
line= lines[i * 2 + 1])
else:
Log.error("version not supported {{version}}", version=self.cluster.version)
if self.debug:
Log.note("{{num}} documents added", num= len(items))
def random_id():
return Random.hex(40)
def random_id():
return Random.hex(40)
def extend(self, records):
"""
records - MUST HAVE FORM OF
[{"value":value}, ... {"value":value}] OR
[{"json":json}, ... {"json":json}]
OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
"""
lines = []
try:
for r in records:
id = r.get("id")
if id == None:
id = Random.hex(40)
if "json" in r:
# if id != coalesce(wrap(convert.json2value(r["json"])).value._id, id):
# Log.error("expecting _id to match")
json = r["json"]
elif "value" in r:
# if id != coalesce(wrap(r).value._id, id):
# Log.error("expecting _id to match")
json = convert.value2json(r["value"])
else:
json = None
Log.error(
"Expecting every record given to have \"value\" or \"json\" property"
)
lines.append('{"index":{"_id": ' + convert.value2json(id) +
'}}')
lines.append(json)
del records
if not lines:
return
try:
data_bytes = "\n".join(lines) + "\n"
data_bytes = data_bytes.encode("utf8")
except Exception, e:
Log.error("can not make request body from\n{{lines|indent}}",
lines=lines,
cause=e)
response = self.cluster._post(self.path + "/_bulk",
data=data_bytes,
headers={"Content-Type": "text"},
timeout=self.settings.timeout)
items = response["items"]
for i, item in enumerate(items):
if self.cluster.version.startswith("0.90."):
if not item.index.ok:
Log.error("{{error}} while loading line:\n{{line}}",
error=item.index.error,
line=lines[i * 2 + 1])
elif any(map(self.cluster.version.startswith,
["1.4.", "1.5."])):
if item.index.status not in [200, 201]:
Log.error(
"{{num}} {{error}} while loading line into {{index}}:\n{{line}}",
num=item.index.status,
error=item.index.error,
line=lines[i * 2 + 1],
index=self.settings.index)
else:
Log.error("version not supported {{version}}",
version=self.cluster.version)
if self.debug:
Log.note("{{num}} documents added", num=len(items))