Exemplo n.º 1
0
 def add_alias(self, alias=None):
     if alias:
         self.cluster_state = None
         self.cluster._post("/_aliases",
                            data=convert.unicode2utf8(
                                convert.value2json({
                                    "actions": [{
                                        "add": {
                                            "index": self.settings.index,
                                            "alias": alias
                                        }
                                    }]
                                })),
                            timeout=coalesce(self.settings.timeout, 30))
     else:
         # SET ALIAS ACCORDING TO LIFECYCLE RULES
         self.cluster_state = None
         self.cluster._post("/_aliases",
                            data=convert.unicode2utf8(
                                convert.value2json({
                                    "actions": [{
                                        "add": {
                                            "index": self.settings.index,
                                            "alias": self.settings.alias
                                        }
                                    }]
                                })),
                            timeout=coalesce(self.settings.timeout, 30))
Exemplo n.º 2
0
def solve():
    try:
        data = convert.json2value(convert.utf82unicode(flask.request.data))
        solved = noop.solve(data)
        response_data = convert.unicode2utf8(convert.value2json(solved))

        return Response(
            response_data,
            direct_passthrough=True,  # FOR STREAMING
            status=200,
            headers={
                "access-control-allow-origin": "*",
                "content-type": "application/json"
            }
        )
    except Exception, e:
        e = Except.wrap(e)
        Log.warning("Could not process", cause=e)
        e = e.as_dict()
        return Response(
            convert.unicode2utf8(convert.value2json(e)),
            status=400,
            headers={
                "access-control-allow-origin": "*",
                "content-type": "application/json"
            }
        )
Exemplo n.º 3
0
    def get_treeherder_job(self):
        try:
            with Timer("Process Request"):
                args = Dict(**flask.request.args)

                # IS THE branch/revision PENDING?

                result = self.get_markup(unwraplist(args.branch),
                                         unwraplist(args.revision),
                                         unwraplist(args.task_id),
                                         unwraplist(args.buildername),
                                         unwraplist(args.timestamp))

                response_data = convert.unicode2utf8(
                    convert.value2json(result))
                return Response(response_data,
                                status=200,
                                headers={
                                    "access-control-allow-origin": "*",
                                    "content-type": "text/plain"
                                })
        except Exception, e:
            e = Except.wrap(e)
            Log.warning("Could not process", cause=e)
            e = e.as_dict()

            return Response(convert.unicode2utf8(convert.value2json(e)),
                            status=400,
                            headers={
                                "access-control-allow-origin": "*",
                                "content-type": "application/json"
                            })
Exemplo n.º 4
0
    def test_rest_get(self):
        settings = self.utils.fill_container({
            "data": [{
                "a": 0,
                "b": 0
            }, {
                "a": 0,
                "b": 1
            }, {
                "a": 1,
                "b": 0
            }, {
                "a": 1,
                "b": 1
            }],
            "query": {
                "from": ""
            }  # DUMMY LINE
        })

        url = URL(self.utils.service_url)
        url.path = "json/" + settings.index
        url.query = {"a": 1}

        response = self.utils.try_till_response(str(url), data=b"")
        self.assertEqual(response.status_code, 200)

        # ORDER DOES NOT MATTER, TEST EITHER
        expected1 = convert.unicode2utf8(
            convert.value2json([{
                "a": 1,
                "b": 0
            }, {
                "a": 1,
                "b": 1
            }],
                               pretty=True))
        expected2 = convert.unicode2utf8(
            convert.value2json([{
                "a": 1,
                "b": 1
            }, {
                "a": 1,
                "b": 0
            }],
                               pretty=True))

        try:
            self.assertEqual(response.all_content, expected1)
        except Exception:
            self.assertEqual(response.all_content, expected2)
Exemplo n.º 5
0
    def put(self, path, **kwargs):
        url = self.settings.host + ":" + unicode(self.settings.port) + path

        data = kwargs.get(b'data')
        if data == None:
            pass
        elif isinstance(data, Mapping):
            kwargs[b'data'] = data = convert.unicode2utf8(convert.value2json(data))
        elif not isinstance(kwargs["data"], str):
            Log.error("data must be utf8 encoded string")

        if self.debug:
            sample = kwargs.get(b'data', "")[:300]
            Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
        # try:
            response = http.put(url, **kwargs)
            if response.status_code not in [200]:
                Log.error(response.reason+": "+response.all_content)
            if self.debug:
                Log.note("response: {{response}}",  response= utf82unicode(response.all_content)[0:300:])

            details = mo_json.json2value(utf82unicode(response.content))
            if details.error:
                Log.error(convert.quote2string(details.error))
            if details._shards.failed > 0:
                Log.error("Shard failures {{failures|indent}}",
                          failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
                          )
            return details
Exemplo n.º 6
0
def get_active_data(settings):
    query = {
    "limit": 100000,
    "from": "unittest",
    "where": {"and": [
        {"eq": {"result.ok": False}},
        {"gt": {"run.timestamp": RECENT.milli}}
    ]},
    "select": [
        "result.ok",
        "build.branch",
        "build.platform",
        "build.release",
        "build.revision",
        "build.type",
        "build.revision",
        "build.date",
        "run.timestamp",
        "run.suite",
        "run.chunk",
        "result.test",
        "run.stats.status.test_status"
    ],
    "format": "table"
    }
    result = http.post("http://activedata.allizom.org/query", data=convert.unicode2utf8(convert.value2json(query)))

    query_result = convert.json2value(convert.utf82unicode(result.all_content))

    tab = convert.table2tab(query_result.header, query_result.data)
    File(settings.output.activedata).write(tab)
Exemplo n.º 7
0
    def test_multiple_agg_on_same_field(self):
        if self.not_real_service():
            return

        test = wrap({
            "query": {
                "from": {
                    "type": "elasticsearch",
                    "settings": {
                        "host": ES_CLUSTER_LOCATION,
                        "index": "unittest",
                        "type": "test_result"
                    }
                },
                "select": [{
                    "name": "max_bytes",
                    "value": "run.stats.bytes",
                    "aggregate": "max"
                }, {
                    "name": "count",
                    "value": "run.stats.bytes",
                    "aggregate": "count"
                }]
            }
        })

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 8
0
def get_raw_json(path):
    active_data_timer = Timer("total duration")
    body = flask.request.get_data()
    try:
        with active_data_timer:
            args = wrap(Data(**flask.request.args))
            limit = args.limit if args.limit else 10
            args.limit = None
            frum = wrap_from(path)
            result = jx.run(
                {
                    "from": path,
                    "where": {
                        "eq": args
                    },
                    "limit": limit,
                    "format": "list"
                }, frum)

            if isinstance(
                    result, Container
            ):  #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
                result = result.format("list")

        result.meta.active_data_response_time = active_data_timer.duration

        response_data = convert.unicode2utf8(
            convert.value2json(result.data, pretty=True))
        Log.note("Response is {{num}} bytes", num=len(response_data))
        return Response(response_data, status=200)
    except Exception, e:
        e = Except.wrap(e)
        return _send_error(active_data_timer, body, e)
Exemplo n.º 9
0
    def test_branch_count(self):
        if self.not_real_service():
            return

        test = wrap({"query": {
            "from": {
                "type": "elasticsearch",
                "settings": {
                    "host": ES_CLUSTER_LOCATION,
                    "index": "unittest",
                    "type": "test_result"
                }
            },
            "select": [
                {"aggregate": "count"},
            ],
            "edges": [
                "build.branch"
            ],
            "where": {"or": [
                {"missing": "build.id"}
                # {"gte": {"timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000}}
            ]},
            "format": "table"
        }})

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 10
0
    def set_refresh_interval(self, seconds):
        if seconds <= 0:
            interval = -1
        else:
            interval = unicode(seconds) + "s"

        if self.cluster.version.startswith("0.90."):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'
            )

            result = convert.json2value(utf82unicode(response.all_content))
            if not result.ok:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}')
            )

            result = convert.json2value(utf82unicode(response.all_content))
            if not result.acknowledged:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        else:
            Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
Exemplo n.º 11
0
    def set_refresh_interval(self, seconds):
        if seconds <= 0:
            interval = -1
        else:
            interval = unicode(seconds) + "s"

        if self.cluster.version.startswith("0.90."):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'
            )

            result = convert.json2value(utf82unicode(response.all_content))
            if not result.ok:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}')
            )

            result = convert.json2value(utf82unicode(response.all_content))
            if not result.acknowledged:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        else:
            Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
Exemplo n.º 12
0
def store_data(path):
    try:
        request = flask.request
        auth = request.headers.get('Authorization')

        if not auth:
            # USE PATTERN MATCHING AUTH
            for c in all_creds:
                if c.path == path:
                    return store_public_data(path, c)
            raise Log.error(
                "No authentication provided.  path={{path}} data.length={{length}}",
                path=path,
                length=len(request.get_data()),
            )

        try:
            receiver = Receiver(
                lookup_credentials,
                auth,
                request.url,
                request.method,
                content=request.get_data(),
                content_type=request.headers['Content-Type'],
                seen_nonce=seen_nonce
            )
        except Exception, e:
            e = Except.wrap(e)
            raise Log.error(
                "Authentication failed.  path={{path}} data.length={{length}}\n{{auth|indent}}",
                path=path,
                length=len(request.get_data()),
                auth=auth,
                cause=e
            )

        permissions = lookup_user(receiver.parsed_header["id"])
        if path not in listwrap(permissions.resources):
            Log.error("{{user}} not allowed access to {{resource}}", user=permissions.hawk.id, resource=path)

        link, id = submit_data(path, permissions, request.json)

        response_content = convert.unicode2utf8(convert.value2json({
            "link": link,
            "etl": {"id": id}
        }))
        receiver.respond(
            content=response_content,
            content_type=RESPONSE_CONTENT_TYPE
        )

        return Response(
            response_content,
            status=200,
            headers={
                b'Server-Authorization': receiver.response_header,
                b'content-type': RESPONSE_CONTENT_TYPE
            }
        )
Exemplo n.º 13
0
def post_json(url, **kwargs):
    """
    ASSUME RESPONSE IN IN JSON
    """
    if b"json" in kwargs:
        kwargs[b"data"] = convert.unicode2utf8(convert.value2json(kwargs[b"json"]))
    elif b'data':
        kwargs[b"data"] = convert.unicode2utf8(convert.value2json(kwargs[b"data"]))
    else:
        Log.error("Expecting `json` parameter")

    response = post(url, **kwargs)
    c = response.content
    try:
        details = convert.json2value(convert.utf82unicode(c))
    except Exception, e:
        Log.error("Unexpected return value {{content}}", content=c, cause=e)
Exemplo n.º 14
0
def default_page(path):
    return Response(
        convert.unicode2utf8(ERROR_PAGE),
        status=400,
        headers={
            "access-control-allow-origin": "*",
            "content-type": "text/html"
        }
    )
Exemplo n.º 15
0
def post_json(url, **kwargs):
    """
    ASSUME RESPONSE IN IN JSON
    """
    kwargs["data"] = convert.unicode2utf8(convert.value2json(kwargs["data"]))

    response = post(url, **kwargs)
    c=response.all_content
    return convert.json2value(convert.utf82unicode(c))
Exemplo n.º 16
0
    def test_missing_auth(self):
        # MAKE SOME DATA
        data = {
            "constant": "this is a test",
            "random-data": convert.bytes2base64(Random.bytes(100))
        }

        response = requests.post(settings.bad_url, data=convert.unicode2utf8(convert.value2json(data)))
        self.assertEqual(response.status_code, 403)
Exemplo n.º 17
0
def post_json(url, **kwargs):
    """
    ASSUME RESPONSE IN IN JSON
    """
    kwargs["data"] = convert.unicode2utf8(convert.value2json(kwargs["data"]))

    response = post(url, **kwargs)
    c = response.all_content
    return convert.json2value(convert.utf82unicode(c))
Exemplo n.º 18
0
    def test_timing(self):
        if self.not_real_service():
            return

        test = wrap({
            "query": {
                "from": {
                    "type": "elasticsearch",
                    "settings": {
                        "host": ES_CLUSTER_LOCATION,
                        "index": "unittest",
                        "type": "test_result"
                    }
                },
                "select": [{
                    "name": "count",
                    "value": "run.duration",
                    "aggregate": "count"
                }, {
                    "name": "total",
                    "value": "run.duration",
                    "aggregate": "sum"
                }],
                "edges": [{
                    "name": "chunk",
                    "value": ["run.suite", "run.chunk"]
                }, "result.ok"],
                "where": {
                    "and": [{
                        "lt": {
                            "timestamp": Date.floor(Date.now()).milli / 1000
                        }
                    }, {
                        "gte": {
                            "timestamp":
                            Date.floor(Date.now() - (Duration.DAY * 7),
                                       Duration.DAY).milli / 1000
                        }
                    }]
                },
                "format":
                "cube",
                "samples": {
                    "limit": 30
                }
            }
        })

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 19
0
def post_json(url, **kwargs):
    """
    ASSUME RESPONSE IN IN JSON
    """
    if b"json" in kwargs:
        kwargs[b"data"] = convert.unicode2utf8(
            convert.value2json(kwargs[b"json"]))
    elif b'data' in kwargs:
        kwargs[b"data"] = convert.unicode2utf8(
            convert.value2json(kwargs[b"data"]))
    else:
        Log.error("Expecting `json` parameter")

    response = post(url, **kwargs)
    c = response.content
    try:
        details = mo_json.json2value(convert.utf82unicode(c))
    except Exception, e:
        Log.error("Unexpected return value {{content}}", content=c, cause=e)
Exemplo n.º 20
0
    def write(self, key, value, disable_zip=False):
        if key.endswith(".json") or key.endswith(".zip"):
            Log.error("Expecting a pure key")

        try:
            if hasattr(value, "read"):
                if disable_zip:
                    storage = self.bucket.new_key(key + ".json")
                    string_length = len(value)
                else:
                    storage = self.bucket.new_key(key + ".json.gz")
                    string_length = len(value)
                    value = convert.bytes2zip(value)
                file_length = len(value)
                Log.note(
                    "Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",
                    file_length=file_length,
                    string_length=string_length)
                value.seek(0)
                storage.set_contents_from_file(value)

                if self.settings.public:
                    storage.set_acl('public-read')
                return

            if len(value) > 20 * 1000 and not disable_zip:
                self.bucket.delete_key(key + ".json")
                self.bucket.delete_key(key + ".json.gz")
                if isinstance(value, str):
                    value = convert.bytes2zip(value)
                    key += ".json.gz"
                else:
                    value = convert.bytes2zip(convert.unicode2utf8(value))
                    key += ".json.gz"

            else:
                self.bucket.delete_key(key + ".json.gz")
                if isinstance(value, str):
                    key += ".json"
                else:
                    key += ".json"

            storage = self.bucket.new_key(key)
            storage.set_contents_from_string(value)

            if self.settings.public:
                storage.set_acl('public-read')
        except Exception, e:
            Log.error(
                "Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
                key=key,
                bucket=self.bucket.name,
                bytes=len(value),
                cause=e)
Exemplo n.º 21
0
    def save(self, query):
        query.meta = None
        json = convert.value2json(query)
        hash = convert.unicode2utf8(json)

        # TRY MANY HASHES AT ONCE
        hashes = [None] * HASH_BLOCK_SIZE
        for i in range(HASH_BLOCK_SIZE):
            hash = hashlib.sha1(hash).digest()
            hashes[i] = hash

        short_hashes = [
            convert.bytes2base64(h[0:6]).replace("/", "_") for h in hashes
        ]
        available = {h: True for h in short_hashes}

        existing = self.es.query({
            "from": {
                "type": "elasticsearch",
                "settings": self.es.settings
            },
            "where": {
                "terms": {
                    "hash": short_hashes
                }
            },
            "meta": {
                "timeout": "2second"
            }
        })

        for e in Cube(select=existing.select,
                      edges=existing.edges,
                      data=existing.data).values():
            if e.query == json:
                return e.hash
            available[e.hash] = False

        # THIS WILL THROW AN ERROR IF THERE ARE NONE, HOW UNLUCKY!
        best = [h for h in short_hashes if available[h]][0]

        self.queue.add({
            "id": best,
            "value": {
                "hash": best,
                "create_time": Date.now(),
                "last_used": Date.now(),
                "query": json
            }
        })

        Log.note("Saved query as {{hash}}", hash=best)

        return best
Exemplo n.º 22
0
def _exit():
    Log.note("Got request to shutdown")
    shutdown = flask.request.environ.get('werkzeug.server.shutdown')
    if shutdown:
        shutdown()
    else:
        Log.warning("werkzeug.server.shutdown does not exist")

    return Response(convert.unicode2utf8(OVERVIEW),
                    status=400,
                    headers={"Content-Type": "text/html"})
Exemplo n.º 23
0
    def write(self, key, value, disable_zip=False):
        if key.endswith(".json") or key.endswith(".zip"):
            Log.error("Expecting a pure key")

        try:
            if hasattr(value, "read"):
                if disable_zip:
                    storage = self.bucket.new_key(key + ".json")
                    string_length = len(value)
                else:
                    storage = self.bucket.new_key(key + ".json.gz")
                    string_length = len(value)
                    value = convert.bytes2zip(value)
                file_length = len(value)
                Log.note("Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",  file_length= file_length,  string_length=string_length)
                value.seek(0)
                storage.set_contents_from_file(value)

                if self.settings.public:
                    storage.set_acl('public-read')
                return

            if len(value) > 20 * 1000 and not disable_zip:
                self.bucket.delete_key(key + ".json")
                self.bucket.delete_key(key + ".json.gz")
                if isinstance(value, str):
                    value = convert.bytes2zip(value)
                    key += ".json.gz"
                else:
                    value = convert.bytes2zip(convert.unicode2utf8(value))
                    key += ".json.gz"

            else:
                self.bucket.delete_key(key + ".json.gz")
                if isinstance(value, str):
                    key += ".json"
                else:
                    key += ".json"

            storage = self.bucket.new_key(key)
            storage.set_contents_from_string(value)

            if self.settings.public:
                storage.set_acl('public-read')
        except Exception, e:
            Log.error(
                "Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
                key=key,
                bucket=self.bucket.name,
                bytes=len(value),
                cause=e
            )
Exemplo n.º 24
0
    def test_simple_query(self):
        if self.not_real_service():
            return

        query = convert.unicode2utf8(convert.value2json({"from": "unittest"}))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 25
0
def post_json(url, **kwargs):
    """
    ASSUME RESPONSE IN IN JSON
    """
    if b"json" in kwargs:
        kwargs[b"data"] = convert.unicode2utf8(convert.value2json(kwargs[b"json"]))
    elif b'data' in kwargs:
        kwargs[b"data"] = convert.unicode2utf8(convert.value2json(kwargs[b"data"]))
    else:
        Log.error("Expecting `json` parameter")

    response = post(url, **kwargs)
    c = response.content
    try:
        details = mo_json.json2value(convert.utf82unicode(c))
    except Exception as e:
        Log.error("Unexpected return value {{content}}", content=c, cause=e)

    if response.status_code not in [200, 201]:
        Log.error("Bad response", cause=Except.wrap(details))

    return details
Exemplo n.º 26
0
def find_query(hash):
    """
    FIND QUERY BY HASH, RETURN Response OBJECT
    :param hash:
    :return: Response OBJECT
    """
    try:
        hash = hash.split("/")[0]
        query = query_finder.find(hash)

        if not query:
            return Response(b'{"type": "ERROR", "template": "not found"}',
                            status=404)
        else:
            return Response(convert.unicode2utf8(query), status=200)
    except Exception, e:
        e = Except.wrap(e)
        Log.warning("problem finding query with hash={{hash}}",
                    hash=hash,
                    cause=e)
        return Response(convert.unicode2utf8(convert.value2json(e)),
                        status=400)
Exemplo n.º 27
0
    def get_treeherder_job(self):
        try:
            with Timer("Process Request"):
                args = Dict(**flask.request.args)

                # IS THE branch/revision PENDING?

                result = self.get_markup(
                    unwraplist(args.branch),
                    unwraplist(args.revision),
                    unwraplist(args.task_id),
                    unwraplist(args.buildername),
                    unwraplist(args.timestamp)
                )

                response_data = convert.unicode2utf8(convert.value2json(result))
                return Response(
                    response_data,
                    status=200,
                    headers={
                        "access-control-allow-origin": "*",
                        "content-type": "text/plain"
                    }
                )
        except Exception, e:
            e = Except.wrap(e)
            Log.warning("Could not process", cause=e)
            e = e.as_dict()

            return Response(
                convert.unicode2utf8(convert.value2json(e)),
                status=400,
                headers={
                    "access-control-allow-origin": "*",
                    "content-type": "application/json"
                }
            )
    def test_save_then_load(self):

        test = {
            "data": [{
                "a": "b"
            }],
            "query": {
                "meta": {
                    "save": True
                },
                "from": TEST_TABLE,
                "select": "a"
            },
            "expecting_list": {
                "meta": {
                    "format": "list"
                },
                "data": ["b"]
            }
        }

        settings = self.utils.fill_container(test)

        bytes = unicode2utf8(
            value2json({
                "from": settings.index,
                "select": "a",
                "format": "list"
            }))
        expected_hash = convert.bytes2base64(
            hashlib.sha1(bytes).digest()[0:6]).replace("/", "_")
        wrap(test).expecting_list.meta.saved_as = expected_hash

        self.utils.send_queries(test)

        # ENSURE THE QUERY HAS BEEN INDEXED
        container = elasticsearch.Index(index="saved_queries", kwargs=settings)
        container.flush()
        Till(seconds=5).wait()

        url = URL(self.utils.service_url)

        response = self.utils.try_till_response(url.scheme + "://" + url.host +
                                                ":" + unicode(url.port) +
                                                "/find/" + expected_hash,
                                                data=b'')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.all_content, bytes)
Exemplo n.º 29
0
 def add_alias(self, alias=None):
     if alias:
         self.cluster_state = None
         self.cluster._post(
             "/_aliases",
             data=convert.unicode2utf8(convert.value2json({
                 "actions": [
                     {"add": {"index": self.settings.index, "alias": alias}}
                 ]
             })),
             timeout=coalesce(self.settings.timeout, 30)
         )
     else:
         # SET ALIAS ACCORDING TO LIFECYCLE RULES
         self.cluster_state = None
         self.cluster._post(
             "/_aliases",
             data=convert.unicode2utf8(convert.value2json({
                 "actions": [
                     {"add": {"index": self.settings.index, "alias": self.settings.alias}}
                 ]
             })),
             timeout=coalesce(self.settings.timeout, 30)
         )
Exemplo n.º 30
0
    def execute_query(self, query):
        query = wrap(query)

        try:
            query = convert.unicode2utf8(convert.value2json(query))
            # EXECUTE QUERY
            response = self.try_till_response(self.service_url, data=query)

            if response.status_code != 200:
                error(response)
            result = convert.json2value(convert.utf82unicode(response.all_content))

            return result
        except Exception, e:
            Log.error("Failed query", e)
Exemplo n.º 31
0
    def post(self, path, **kwargs):
        url = self.settings.host + ":" + unicode(self.settings.port) + path

        try:
            wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"

            data = kwargs.get(b'data')
            if data == None:
                pass
            elif isinstance(data, Mapping):
                kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
            elif not isinstance(kwargs["data"], str):
                Log.error("data must be utf8 encoded string")

            if self.debug:
                sample = kwargs.get(b'data', "")[:300]
                Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)

            if self.debug:
                Log.note("POST {{url}}", url=url)
            response = http.post(url, **kwargs)
            if response.status_code not in [200, 201]:
                Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
            if self.debug:
                Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
            details = mo_json.json2value(utf82unicode(response.content))
            if details.error:
                Log.error(convert.quote2string(details.error))
            if details._shards.failed > 0:
                Log.error("Shard failures {{failures|indent}}",
                    failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
                )
            return details
        except Exception, e:
            if url[0:4] != "http":
                suggestion = " (did you forget \"http://\" prefix on the host name?)"
            else:
                suggestion = ""

            if kwargs.get("data"):
                Log.error(
                    "Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
                    url=url,
                    body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
                    cause=e
                )
            else:
                Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
Exemplo n.º 32
0
    def post(self, path, **kwargs):
        url = self.settings.host + ":" + unicode(self.settings.port) + path

        try:
            wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"

            data = kwargs.get(b'data')
            if data == None:
                pass
            elif isinstance(data, Mapping):
                kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
            elif not isinstance(kwargs["data"], str):
                Log.error("data must be utf8 encoded string")

            if self.debug:
                sample = kwargs.get(b'data', "")[:300]
                Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)

            if self.debug:
                Log.note("POST {{url}}", url=url)
            response = http.post(url, **kwargs)
            if response.status_code not in [200, 201]:
                Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
            if self.debug:
                Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
            details = convert.json2value(utf82unicode(response.content))
            if details.error:
                Log.error(convert.quote2string(details.error))
            if details._shards.failed > 0:
                Log.error("Shard failures {{failures|indent}}",
                    failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
                )
            return details
        except Exception, e:
            if url[0:4] != "http":
                suggestion = " (did you forget \"http://\" prefix on the host name?)"
            else:
                suggestion = ""

            if kwargs.get("data"):
                Log.error(
                    "Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
                    url=url,
                    body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
                    cause=e
                )
            else:
                Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
Exemplo n.º 33
0
def _send_error(active_data_timer, body, e):
    record_request(flask.request, None, body, e)
    Log.warning("Could not process\n{{body}}",
                body=body.decode("latin1"),
                cause=e)
    e = e.__data__()
    e.meta.timing.total = active_data_timer.duration.seconds

    # REMOVE TRACES, BECAUSE NICER TO HUMANS
    # def remove_trace(e):
    #     e.trace = e.trace[0:1:]
    #     for c in listwrap(e.cause):
    #         remove_trace(c)
    # remove_trace(e)

    return Response(convert.unicode2utf8(convert.value2json(e)), status=400)
Exemplo n.º 34
0
    def get(*args, **kwargs):
        body = kwargs.get("data")

        if not body:
            return wrap({"status_code": 400})

        text = convert.utf82unicode(body)
        text = replace_vars(text)
        data = convert.json2value(text)
        result = jx.run(data)
        output_bytes = convert.unicode2utf8(convert.value2json(result))
        return wrap({
            "status_code": 200,
            "all_content": output_bytes,
            "content": output_bytes
        })
Exemplo n.º 35
0
    def test_longest_running_tests(self):
        test = wrap({
            "query": {
                "sort": {
                    "sort": -1,
                    "field": "avg"
                },
                "from": {
                    "from":
                    "unittest",
                    "where": {
                        "and": [{
                            "gt": {
                                "build.date": "1439337600"
                            }
                        }]
                    },
                    "groupby": [
                        "build.platform", "build.type", "run.suite",
                        "result.test"
                    ],
                    "select": [{
                        "aggregate": "avg",
                        "name": "avg",
                        "value": "result.duration"
                    }],
                    "format":
                    "table",
                    "limit":
                    100
                },
                "limit": 100,
                "format": "list"
            }
        })
        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 36
0
def store_public_data(path, permissions):
    """
    :param path: THE BUCKET TO USE
    :param permissions: THE DATA PATTERN EXPECTED
    :return: LINK TO DATA
    """
    try:
        request = flask.request

        if request.content_length > permissions.max_size or len(request.get_data()) > permissions.max_size:
            Log.error("Not acceptable, too big")

        json_data = wrap(request.json)
        for k, _ in permissions.pattern.leaves():
            if not json_data[k]:
                Log.error("Not acceptable\n{{data|json}}", data=json_data)

        link, id = submit_data(path, permissions, request.json)

        response_content = convert.unicode2utf8(convert.value2json({
            "link": link,
            "etl": {"id": id}
        }))

        return Response(
            response_content,
            status=200,
            headers={
                'content-type': RESPONSE_CONTENT_TYPE
            }
        )

    except Exception, e:
        e = Except.wrap(e)
        Log.warning("Error", cause=e)

        return Response(
            RESPONSE_4XX,
            status=403,
            headers={
                'content-type': "text/plain"
            }
        )
Exemplo n.º 37
0
    def test_failures_by_directory(self):
        if self.not_real_service():
            return

        test = wrap({"query": {
            "from": {
                "type": "elasticsearch",
                "settings": {
                    "host": ES_CLUSTER_LOCATION,
                    "index": "unittest",
                    "type": "test_result"
                }
            },
            "select": [
                {
                    "aggregate": "count"
                }
            ],
            "edges": [
                "result.test",
                "result.ok"
            ],
            "where": {
                "prefix": {
                    "result.test": "/"
                }
            },
            "format": "table"
        }})

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Exemplo n.º 38
0
    def set_refresh_interval(self, seconds, **kwargs):
        """
        :param seconds:  -1 FOR NO REFRESH
        :param kwargs: ANY OTHER REQUEST PARAMETERS
        :return: None
        """
        if seconds <= 0:
            interval = -1
        else:
            interval = unicode(seconds) + "s"

        if self.cluster.version.startswith("0.90."):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}',
                **kwargs
            )

            result = mo_json.json2value(utf82unicode(response.all_content))
            if not result.ok:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'),
                **kwargs
            )

            result = mo_json.json2value(utf82unicode(response.all_content))
            if not result.acknowledged:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        else:
            Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
Exemplo n.º 39
0
    def set_refresh_interval(self, seconds, **kwargs):
        """
        :param seconds:  -1 FOR NO REFRESH
        :param kwargs: ANY OTHER REQUEST PARAMETERS
        :return: None
        """
        if seconds <= 0:
            interval = -1
        else:
            interval = unicode(seconds) + "s"

        if self.cluster.version.startswith("0.90."):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}',
                **kwargs
            )

            result = mo_json.json2value(utf82unicode(response.all_content))
            if not result.ok:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
            response = self.cluster.put(
                "/" + self.settings.index + "/_settings",
                data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'),
                **kwargs
            )

            result = mo_json.json2value(utf82unicode(response.all_content))
            if not result.acknowledged:
                Log.error("Can not set refresh interval ({{error}})", {
                    "error": utf82unicode(response.all_content)
                })
        else:
            Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
Exemplo n.º 40
0
    def send_queries(self, subtest):
        subtest = wrap(subtest)

        try:
            # EXECUTE QUERY
            num_expectations = 0
            for k, v in subtest.items():
                if k.startswith("expecting_"):  # WHAT FORMAT ARE WE REQUESTING
                    format = k[len("expecting_"):]
                elif k == "expecting":  # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS)
                    format = None
                else:
                    continue

                num_expectations += 1
                expected = v

                subtest.query.format = format
                subtest.query.meta.testing = True  # MARK ALL QUERIES FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION
                query = convert.unicode2utf8(convert.value2json(subtest.query))
                # EXECUTE QUERY
                response = self.try_till_response(self.service_url, data=query)

                if response.status_code != 200:
                    error(response)
                result = convert.json2value(
                    convert.utf82unicode(response.all_content))

                # HOW TO COMPARE THE OUT-OF-ORDER DATA?
                compare_to_expected(subtest.query, result, expected)
                Log.note("Test result compares well")
            if num_expectations == 0:
                Log.error(
                    "Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses",
                    {"name": subtest.name})
        except Exception, e:
            Log.error("Failed test {{name|quote}}", {"name": subtest.name}, e)
Exemplo n.º 41
0
def get_bugs(settings):
    request_bugs = convert.unicode2utf8(convert.value2json({
        "query": {"filtered": {
            "query": {"match_all": {}},
            "filter": {"and": [
                {"term": {"keyword": "intermittent-failure"}},
                {"range": {"expires_on": {"gt": Date.now().milli}}},
                {"range": {"modified_ts": {"gt": RECENT.milli}}}
            ]}
        }},
        "from": 0,
        "size": 200000,
        "sort": [],
        "facets": {},
        "fields": ["bug_id", "bug_status", "short_desc", "status_whiteboard"]
    }))
    result = http.post(settings.bugs.url, data=request_bugs).all_content
    bugs = UniqueIndex(["bug_id"], convert.json2value(convert.utf82unicode(result)).hits.hits.fields)

    for i, b in enumerate(bugs):
        try:
            parse_short_desc(b)
        except Exception, e:
            Log.warning("can not parse {{bug_id}} {{short_desc}}", bug_id=b.bug_id, short_desc=b.short_desc, cause=e)
Exemplo n.º 42
0
from pyLibrary.dot import unwrap
from pyLibrary.maths.randoms import Random
from modatasubmission import Client


settings = jsons.ref.get("file://~/MoDataSubmissionClient.json")


data={
    "constant": "this is a test",
    "random-data": convert.bytes2base64(Random.bytes(100))
}
link, id = Client(settings.url, unwrap(settings.hawk)).send(data)
Log.note("Success!  Located at {{link}} id={{id}}", link=link, id=id)


data = convert.unicode2utf8(convert.value2json(settings.example))

response = requests.post(
    settings.url,
    data=data,
    headers={
        'Content-Type': b'application/json'
    }
)
if response.status_code != 200:
    Log.error("Expecting a pass")

details = convert.json2value(convert.utf82unicode(response.content))
Log.note("Success!  Located at {{link}} id={{id}}", link=details.link, id=details.etl.id)
Exemplo n.º 43
0
    result = http.post(settings.bugs.url, data=request_bugs).all_content
    bugs = UniqueIndex(["bug_id"], convert.json2value(convert.utf82unicode(result)).hits.hits.fields)

    for i, b in enumerate(bugs):
        try:
            parse_short_desc(b)
        except Exception, e:
            Log.warning("can not parse {{bug_id}} {{short_desc}}", bug_id=b.bug_id, short_desc=b.short_desc, cause=e)

    request_comments = convert.unicode2utf8(convert.value2json({
        "query": {"filtered": {
            "query": {"match_all": {}},
            "filter": {"and":[
                {"terms": {"bug_id": bugs.keys()}},
                {"range": {"modified_ts": {"gt": RECENT.milli}}}
            ]}
        }},
        "from": 0,
        "size": 200000,
        "sort": [],
        "facets": {},
        "fields": ["bug_id", "modified_by", "modified_ts", "comment"]
    }))

    comments = convert.json2value(convert.utf82unicode(http.post(settings.comments.url, data=request_comments).all_content)).hits.hits.fields

    results = []
    for c in comments:
        errors = parse_comment(bugs[c.bug_id], c)
        results.extend(errors)

    tab = convert.list2tab(results)
Exemplo n.º 44
0
def _default(path):
    record_request(flask.request, None, flask.request.get_data(), None)

    return Response(convert.unicode2utf8(OVERVIEW),
                    status=200,
                    headers={"Content-Type": "text/html"})
Exemplo n.º 45
0
def query(path):
    with CProfiler():
        try:
            with Timer("total duration") as query_timer:
                preamble_timer = Timer("preamble")
                with preamble_timer:
                    if flask.request.headers.get("content-length",
                                                 "") in ["", "0"]:
                        # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
                        return Response(BLANK,
                                        status=400,
                                        headers={"Content-Type": "text/html"})
                    elif int(flask.request.headers["content-length"]
                             ) > QUERY_SIZE_LIMIT:
                        Log.error("Query is too large")

                    request_body = flask.request.get_data().strip()
                    text = convert.utf82unicode(request_body)
                    text = replace_vars(text, flask.request.args)
                    data = convert.json2value(text)
                    record_request(flask.request, data, None, None)
                    if data.meta.testing:
                        _test_mode_wait(data)

                translate_timer = Timer("translate")
                with translate_timer:
                    if data.sql:
                        data = parse_sql(data.sql)
                    frum = wrap_from(data['from'])
                    result = jx.run(data, frum=frum)

                    if isinstance(
                            result, Container
                    ):  #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
                        result = result.format(data.format)

                save_timer = Timer("save")
                with save_timer:
                    if data.meta.save:
                        try:
                            result.meta.saved_as = save_query.query_finder.save(
                                data)
                        except Exception, e:
                            Log.warning("Unexpected save problem", cause=e)

                result.meta.timing.preamble = Math.round(
                    preamble_timer.duration.seconds, digits=4)
                result.meta.timing.translate = Math.round(
                    translate_timer.duration.seconds, digits=4)
                result.meta.timing.save = Math.round(
                    save_timer.duration.seconds, digits=4)
                result.meta.timing.total = "{{TOTAL_TIME}}"  # TIMING PLACEHOLDER

                with Timer("jsonification") as json_timer:
                    response_data = convert.unicode2utf8(
                        convert.value2json(result))

            with Timer("post timer"):
                # IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.
                # WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
                timing_replacement = b'"total": ' + str(Math.round(query_timer.duration.seconds, digits=4)) +\
                                     b', "jsonification": ' + str(Math.round(json_timer.duration.seconds, digits=4))
                response_data = response_data.replace(
                    b'"total": "{{TOTAL_TIME}}"', timing_replacement)
                Log.note("Response is {{num}} bytes in {{duration}}",
                         num=len(response_data),
                         duration=query_timer.duration)

                return Response(
                    response_data,
                    status=200,
                    headers={"Content-Type": result.meta.content_type})
        except Exception, e:
            e = Except.wrap(e)
            return _send_error(query_timer, request_body, e)
Exemplo n.º 46
0
from active_data import record_request, cors_wrapper
from active_data.actions import save_query
from mo_dots import coalesce, join_field, split_field, wrap, listwrap
from pyLibrary import convert
from mo_files import File
from mo_math import Math
from pyLibrary.queries import jx, meta, wrap_from
from pyLibrary.queries.containers import Container, STRUCT
from pyLibrary.queries.meta import TOO_OLD
from mo_testing.fuzzytestcase import assertAlmostEqual
from mo_threads import Till
from mo_times.dates import Date
from mo_times.durations import MINUTE
from mo_times.timer import Timer

BLANK = convert.unicode2utf8(File("active_data/public/error.html").read())
QUERY_SIZE_LIMIT = 10 * 1024 * 1024


@cors_wrapper
def query(path):
    with CProfiler():
        try:
            with Timer("total duration") as query_timer:
                preamble_timer = Timer("preamble")
                with preamble_timer:
                    if flask.request.headers.get("content-length",
                                                 "") in ["", "0"]:
                        # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
                        return Response(BLANK,
                                        status=400,