def post(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate" data = kwargs.get(b'data') if data == None: pass elif isinstance(data, Mapping): kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data)) elif not isinstance(kwargs["data"], str): Log.error("data must be utf8 encoded string") if self.debug: sample = kwargs.get(b'data', "")[:300] Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample) if self.debug: Log.note("POST {{url}}", url=url) response = http.post(url, **kwargs) if response.status_code not in [200, 201]: Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000)) if self.debug: Log.note("response: {{response}}", response=utf82unicode(response.content)[:130]) details = convert.json2value(utf82unicode(response.content)) if details.error: Log.error(convert.quote2string(details.error)) if details._shards.failed > 0: Log.error("Shard failures {{failures|indent}}", failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason) ) return details except Exception, e: if url[0:4] != "http": suggestion = " (did you forget \"http://\" prefix on the host name?)" else: suggestion = "" if kwargs.get("data"): Log.error( "Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}", url=url, body=strings.limit(kwargs["data"], 100 if self.debug else 10000), cause=e ) else: Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def json2value(json_string, params={}, flexible=False, leaves=False): """ :param json_string: THE JSON :param params: STANDARD JSON PARAMS :param flexible: REMOVE COMMENTS :param leaves: ASSUME JSON KEYS ARE DOT-DELIMITED :return: Python value """ if isinstance(json_string, str): Log.error("only unicode json accepted") try: if flexible: # REMOVE """COMMENTS""", # COMMENTS, //COMMENTS, AND \n \r # DERIVED FROM https://github.com/jeads/datasource/blob/master/datasource/bases/BaseHub.py# L58 json_string = re.sub(r"\"\"\".*?\"\"\"", r"\n", json_string, flags=re.MULTILINE) json_string = "\n".join(remove_line_comment(l) for l in json_string.split("\n")) # ALLOW DICTIONARY'S NAME:VALUE LIST TO END WITH COMMA json_string = re.sub(r",\s*\}", r"}", json_string) # ALLOW LISTS TO END WITH COMMA json_string = re.sub(r",\s*\]", r"]", json_string) if params: json_string = expand_template(json_string, params) # LOOKUP REFERENCES value = wrap(json_decoder(json_string)) if leaves: value = wrap_leaves(value) return value except Exception, e: e = Except.wrap(e) if "Expecting '" in e and "' delimiter: line" in e: line_index = int(strings.between(e.message, " line ", " column ")) - 1 column = int(strings.between(e.message, " column ", " ")) - 1 line = json_string.split("\n")[line_index].replace("\t", " ") if column > 20: sample = "..." + line[column - 20:] pointer = " " + (" " * 20) + "^" else: sample = line pointer = (" " * column) + "^" if len(sample) > 43: sample = sample[:43] + "..." Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n") base_str = unicode2utf8(strings.limit(json_string, 1000)) hexx_str = bytes2hex(base_str, " ") try: char_str = " " + (" ".join(c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str) except Exception: char_str = " " Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e)
def write(self, template, params): if params.get("template"): # DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE self.queue.add({"value": params}) else: template = strings.limit(template, 2000) self.queue.add({"value": {"template": template, "params": params}}, timeout=3*MINUTE) return self
def write(self, template, params): if params.get("template"): # DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE self.queue.add({"value": params}) else: template = strings.limit(template, 2000) self.queue.add({"value": {"template": template, "params": params}}, timeout=3 * MINUTE) return self
def _deep_json_to_string(value, depth): """ :param value: SOME STRUCTURE :param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED :return: FLATTER STRUCTURE """ if isinstance(value, Mapping): if depth == 0: return strings.limit(convert.value2json(value), LOG_STRING_LENGTH) return {k: _deep_json_to_string(v, depth - 1) for k, v in value.items()} elif isinstance(value, list): return strings.limit(convert.value2json(value), LOG_STRING_LENGTH) elif isinstance(value, (float, int, long)): return value elif isinstance(value, basestring): return strings.limit(value, LOG_STRING_LENGTH) else: return strings.limit(convert.value2json(value), LOG_STRING_LENGTH)
def get(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: response = http.get(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130)) details = wrap(convert.json2value(utf82unicode(response.all_content))) if details.error: Log.error(details.error) return details except Exception, e: Log.error("Problem with call to {{url}}", url=url, cause=e)
line_index = int(strings.between(c.message, " line ", " column ")) - 1 column = int(strings.between(c.message, " column ", " ")) - 1 line = json_string.split("\n")[line_index].replace("\t", " ") if column > 20: sample = "..." + line[column - 20:] pointer = " " + (" " * 20) + "^" else: sample = line pointer = (" " * column) + "^" if len(sample) > 43: sample = sample[:43] + "..." Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n") base_str = unicode2utf8(strings.limit(json_string, 1000)) hexx_str = bytes2hex(base_str, " ") try: char_str = " " + " ".join((c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str) except Exception, e: char_str = " " Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e) def string2datetime(value, format=None): return unix2datetime(Date(value, format).unix) def str2datetime(value, format=None): return unix2datetime(Date(value, format).unix)
def extend(self, records): """ records - MUST HAVE FORM OF [{"value":value}, ... {"value":value}] OR [{"json":json}, ... {"json":json}] OPTIONAL "id" PROPERTY IS ALSO ACCEPTED """ if self.settings.read_only: Log.error("Index opened in read only mode, no changes allowed") lines = [] try: for r in records: id = r.get("id") if id == None: id = random_id() if "json" in r: json_bytes = r["json"].encode("utf8") elif "value" in r: json_bytes = convert.value2json(r["value"]).encode("utf8") else: json_bytes = None Log.error("Expecting every record given to have \"value\" or \"json\" property") lines.append(b'{"index":{"_id": ' + convert.value2json(id).encode("utf8") + b'}}') if self.settings.tjson: lines.append(json2typed(json_bytes.decode('utf8')).encode('utf8')) else: lines.append(json_bytes) del records if not lines: return try: data_bytes = b"\n".join(l for l in lines) + b"\n" except Exception, e: Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e) response = self.cluster.post( self.path + "/_bulk", data=data_bytes, headers={"Content-Type": "text"}, timeout=self.settings.timeout, retry=self.settings.retry ) items = response["items"] fails = [] if self.cluster.version.startswith("0.90."): for i, item in enumerate(items): if not item.index.ok: fails.append(i) elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])): for i, item in enumerate(items): if item.index.status not in [200, 201]: fails.append(i) else: Log.error("version not supported {{version}}", version=self.cluster.version) if fails: Log.error("Problems with insert", cause=[ Except( template="{{status}} {{error}} (and {{some}} others) while loading line id={{id}} into index {{index|quote}}:\n{{line}}", status=items[i].index.status, error=items[i].index.error, some=len(fails) - 1, line=strings.limit(lines[fails[0] * 2 + 1], 500 if not self.debug else 100000), index=self.settings.index, id=items[i].index._id ) for i in fails ]) if self.debug: Log.note("{{num}} documents added", num=len(items))
def extend(self, records): """ records - MUST HAVE FORM OF [{"value":value}, ... {"value":value}] OR [{"json":json}, ... {"json":json}] OPTIONAL "id" PROPERTY IS ALSO ACCEPTED """ if self.settings.read_only: Log.error("Index opened in read only mode, no changes allowed") lines = [] try: for r in records: id = r.get("id") if id == None: id = random_id() if "json" in r: json_bytes = r["json"].encode("utf8") elif "value" in r: json_bytes = convert.value2json(r["value"]).encode("utf8") else: json_bytes = None Log.error("Expecting every record given to have \"value\" or \"json\" property") lines.append(b'{"index":{"_id": ' + convert.value2json(id).encode("utf8") + b'}}') if self.settings.tjson: lines.append(json2typed(json_bytes.decode('utf8')).encode('utf8')) else: lines.append(json_bytes) del records if not lines: return with Timer("Add {{num}} documents to {{index}}", {"num": len(lines) / 2, "index":self.settings.index}, debug=self.debug): try: data_bytes = b"\n".join(l for l in lines) + b"\n" except Exception, e: Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e) response = self.cluster.post( self.path + "/_bulk", data=data_bytes, headers={"Content-Type": "text"}, timeout=self.settings.timeout, retry=self.settings.retry ) items = response["items"] fails = [] if self.cluster.version.startswith("0.90."): for i, item in enumerate(items): if not item.index.ok: fails.append(i) elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])): for i, item in enumerate(items): if item.index.status not in [200, 201]: fails.append(i) else: Log.error("version not supported {{version}}", version=self.cluster.version) if fails: Log.error("Problems with insert", cause=[ Except( template="{{status}} {{error}} (and {{some}} others) while loading line id={{id}} into index {{index|quote}}:\n{{line}}", status=items[i].index.status, error=items[i].index.error, some=len(fails) - 1, line=strings.limit(lines[fails[0] * 2 + 1], 500 if not self.debug else 100000), index=self.settings.index, id=items[i].index._id ) for i in fails ]) except Exception, e: if e.message.startswith("sequence item "): Log.error("problem with {{data}}", data=repr(lines[int(e.message[14:16].strip())]), cause=e) Log.error("problem sending to ES", e)
def extend(self, records): """ records - MUST HAVE FORM OF [{"value":value}, ... {"value":value}] OR [{"json":json}, ... {"json":json}] OPTIONAL "id" PROPERTY IS ALSO ACCEPTED """ if self.settings.read_only: Log.error("Index opened in read only mode, no changes allowed") lines = [] try: for r in records: id = r.get("id") if id == None: id = random_id() if "json" in r: # if id != coalesce(wrap(convert.json2value(r["json"])).value._id, id): # Log.error("expecting _id to match") json = r["json"] elif "value" in r: # if id != coalesce(wrap(r).value._id, id): # Log.error("expecting _id to match") json = convert.value2json(r["value"]) else: json = None Log.error("Expecting every record given to have \"value\" or \"json\" property") lines.append('{"index":{"_id": ' + convert.value2json(id) + '}}') if self.settings.tjson: lines.append(json2typed(json)) else: lines.append(json) del records if not lines: return try: data_bytes = "\n".join(lines) + "\n" data_bytes = data_bytes.encode("utf8") except Exception, e: Log.error("can not make request body from\n{{lines|indent}}", lines=lines, cause=e) response = self.cluster.post( self.path + "/_bulk", data=data_bytes, headers={"Content-Type": "text"}, timeout=self.settings.timeout ) items = response["items"] for i, item in enumerate(items): if self.cluster.version.startswith("0.90."): if not item.index.ok: Log.error( "{{error}} while loading line:\n{{line}}", error=item.index.error, line=lines[i * 2 + 1] ) elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])): if item.index.status not in [200, 201]: Log.error( "{{num}} {{error}} while loading line id={{id}} into index {{index|quote}}:\n{{line}}", num=item.index.status, error=item.index.error, line=strings.limit(lines[i * 2 + 1], 300), index=self.settings.index, id=item.index._id ) else: Log.error("version not supported {{version}}", version=self.cluster.version) if self.debug: Log.note("{{num}} documents added", num=len(items))