Пример #1
0
    def extend(self, records):
        """
        records - MUST HAVE FORM OF
            [{"value":value}, ... {"value":value}] OR
            [{"json":json}, ... {"json":json}]
            OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
        """
        lines = []
        try:
            for r in records:
                id = r.get("id")
                if id == None:
                    id = Random.hex(40)

                if "json" in r:
                    json = r["json"]
                elif "value" in r:
                    json = convert.value2json(r["value"])
                else:
                    json = None
                    Log.error("Expecting every record given to have \"value\" or \"json\" property")

                lines.append('{"index":{"_id": ' + convert.value2json(id) + '}}')
                lines.append(json)
            del records

            if not lines:
                return

            try:
                data_bytes = "\n".join(lines) + "\n"
                data_bytes = data_bytes.encode("utf8")
            except Exception, e:
                Log.error("can not make request body from\n{{lines|indent}}",  lines= lines, cause=e)


            response = self.cluster._post(
                self.path + "/_bulk",
                data=data_bytes,
                headers={"Content-Type": "text"},
                timeout=self.settings.timeout
            )
            items = response["items"]

            for i, item in enumerate(items):
                if self.cluster.version.startswith("0.90."):
                    if not item.index.ok:
                        Log.error("{{error}} while loading line:\n{{line}}",
                            error= item.index.error,
                            line= lines[i * 2 + 1])
                elif self.cluster.version.startswith("1.4."):
                    if item.index.status not in [200, 201]:
                        Log.error("{{error}} while loading line:\n{{line}}",
                            error= item.index.error,
                            line= lines[i * 2 + 1])
                else:
                    Log.error("version not supported {{version}}",  version=self.cluster.version)

            if self.debug:
                Log.note("{{num}} documents added",  num= len(items))
Пример #2
0
def random_id():
    return Random.hex(40)
Пример #3
0
def random_id():
    return Random.hex(40)
Пример #4
0
    def extend(self, records):
        """
        records - MUST HAVE FORM OF
            [{"value":value}, ... {"value":value}] OR
            [{"json":json}, ... {"json":json}]
            OPTIONAL "id" PROPERTY IS ALSO ACCEPTED
        """
        lines = []
        try:
            for r in records:
                id = r.get("id")

                if id == None:
                    id = Random.hex(40)

                if "json" in r:
                    # if id != coalesce(wrap(convert.json2value(r["json"])).value._id, id):
                    #     Log.error("expecting _id to match")
                    json = r["json"]
                elif "value" in r:
                    # if id != coalesce(wrap(r).value._id, id):
                    #     Log.error("expecting _id to match")
                    json = convert.value2json(r["value"])
                else:
                    json = None
                    Log.error(
                        "Expecting every record given to have \"value\" or \"json\" property"
                    )

                lines.append('{"index":{"_id": ' + convert.value2json(id) +
                             '}}')
                lines.append(json)
            del records

            if not lines:
                return

            try:
                data_bytes = "\n".join(lines) + "\n"
                data_bytes = data_bytes.encode("utf8")
            except Exception, e:
                Log.error("can not make request body from\n{{lines|indent}}",
                          lines=lines,
                          cause=e)

            response = self.cluster._post(self.path + "/_bulk",
                                          data=data_bytes,
                                          headers={"Content-Type": "text"},
                                          timeout=self.settings.timeout)
            items = response["items"]

            for i, item in enumerate(items):
                if self.cluster.version.startswith("0.90."):
                    if not item.index.ok:
                        Log.error("{{error}} while loading line:\n{{line}}",
                                  error=item.index.error,
                                  line=lines[i * 2 + 1])
                elif any(map(self.cluster.version.startswith,
                             ["1.4.", "1.5."])):
                    if item.index.status not in [200, 201]:
                        Log.error(
                            "{{num}} {{error}} while loading line into {{index}}:\n{{line}}",
                            num=item.index.status,
                            error=item.index.error,
                            line=lines[i * 2 + 1],
                            index=self.settings.index)
                else:
                    Log.error("version not supported {{version}}",
                              version=self.cluster.version)

            if self.debug:
                Log.note("{{num}} documents added", num=len(items))