def get_raw_json(path): active_data_timer = Timer("total duration") body = flask.request.get_data() try: with active_data_timer: args = wrap(Data(**flask.request.args)) limit = args.limit if args.limit else 10 args.limit = None frum = wrap_from(path) result = jx.run( { "from": path, "where": { "eq": args }, "limit": limit, "format": "list" }, frum) if isinstance( result, Container ): #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers result = result.format("list") result.meta.active_data_response_time = active_data_timer.duration response_data = convert.unicode2utf8( convert.value2json(result.data, pretty=True)) Log.note("Response is {{num}} bytes", num=len(response_data)) return Response(response_data, status=200) except Exception, e: e = Except.wrap(e) return _send_error(active_data_timer, body, e)
def wrap(query, schema=None): """ NORMALIZE QUERY SO IT CAN STILL BE JSON """ if isinstance(query, QueryOp) or query == None: return query query = wrap(query) output = QueryOp("from", None) output.format = query.format output.frum = wrap_from(query["from"], schema=schema) if not schema and isinstance(output.frum, Schema): schema = output.frum if not schema and hasattr(output.frum, "schema"): schema = output.frum.schema if query.select or isinstance(query.select, (Mapping, list)): output.select = _normalize_selects(query.select, query.frum, schema=schema) else: if query.edges or query.groupby: output.select = Data(name="count", value=jx_expression("."), aggregate="count", default=0) else: output.select = _normalize_selects(".", query.frum) if query.groupby and query.edges: Log.error( "You can not use both the `groupby` and `edges` clauses in the same query!" ) elif query.edges: output.edges = _normalize_edges(query.edges, schema=schema) output.groupby = Null elif query.groupby: output.edges = Null output.groupby = _normalize_groupby(query.groupby, schema=schema) else: output.edges = Null output.groupby = Null output.where = _normalize_where(query.where, schema=schema) output.window = [_normalize_window(w) for w in listwrap(query.window)] output.having = None output.sort = _normalize_sort(query.sort) output.limit = Math.min(MAX_LIMIT, coalesce(query.limit, DEFAULT_LIMIT)) if not Math.is_integer(output.limit) or output.limit < 0: Log.error("Expecting limit >= 0") output.isLean = query.isLean return output
def allowed_query(data): data = json2value(data) data.edges = None data.groupby = None data.select = {"value": "result.test", "aggregate": "cardinality"} data.format = "list" frum = wrap_from(data['from']) result = jx.run(data, frum=frum) if result.data < 10000: return True else: Log.error("not allowed to groupby result.test")
def wrap(query, schema=None): """ NORMALIZE QUERY SO IT CAN STILL BE JSON """ if isinstance(query, QueryOp) or query == None: return query query = wrap(query) output = QueryOp("from", None) output.format = query.format output.frum = wrap_from(query["from"], schema=schema) if not schema and isinstance(output.frum, Schema): schema = output.frum if not schema and hasattr(output.frum, "schema"): schema = output.frum.schema if query.select or isinstance(query.select, (Mapping, list)): output.select = _normalize_selects(query.select, query.frum, schema=schema) else: if query.edges or query.groupby: output.select = Data(name="count", value=jx_expression("."), aggregate="count", default=0) else: output.select = _normalize_selects(".", query.frum) if query.groupby and query.edges: Log.error("You can not use both the `groupby` and `edges` clauses in the same query!") elif query.edges: output.edges = _normalize_edges(query.edges, schema=schema) output.groupby = Null elif query.groupby: output.edges = Null output.groupby = _normalize_groupby(query.groupby, schema=schema) else: output.edges = Null output.groupby = Null output.where = _normalize_where(query.where, schema=schema) output.window = [_normalize_window(w) for w in listwrap(query.window)] output.having = None output.sort = _normalize_sort(query.sort) output.limit = Math.min(MAX_LIMIT, coalesce(query.limit, DEFAULT_LIMIT)) if not Math.is_integer(output.limit) or output.limit < 0: Log.error("Expecting limit >= 0") output.isLean = query.isLean return output
def __init__(self, query, schema=None): """ NORMALIZE QUERY SO IT CAN STILL BE JSON """ if isinstance(query, Query) or query == None: return object.__init__(self) query = wrap(query) self.format = query.format self.frum = wrap_from(query["from"], schema=schema) select = query.select if isinstance(select, list): names = set() new_select = [] for s in select: ns = _normalize_select(s, schema=schema) if ns.name in names: Log.error("two select have the same name") names.add(ns.name) new_select.append(unwrap(ns)) self.select = wrap(new_select) elif select: self.select = _normalize_select(select, schema=schema) else: if query.edges or query.groupby: self.select = Dict(name="count", value=".", aggregate="count") else: self.select = Dict(name=".", value=".", aggregate="none") if query.groupby and query.edges: Log.error("You can not use both the `groupby` and `edges` clauses in the same query!") elif query.edges: self.edges = _normalize_edges(query.edges, schema=schema) self.groupby = None elif query.groupby: self.edges = None self.groupby = _normalize_groupby(query.groupby, schema=schema) else: self.edges = [] self.groupby = None self.where = _normalize_where(query.where, schema=schema) self.window = [_normalize_window(w) for w in listwrap(query.window)] self.having = None self.sort = _normalize_sort(query.sort) self.limit = Math.min(MAX_LIMIT, coalesce(query.limit, DEFAULT_LIMIT)) if not Math.is_integer(self.limit) or self.limit < 0: Log.error("Expecting limit >= 0") self.isLean = query.isLean # DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN # THE from SOURCE IS. # TODO: IGNORE REACHING INTO THE NON-NESTED TYPES if isinstance(self.frum, list): if not qb: _late_import() columns = qb.get_columns(self.frum) elif isinstance(self.frum, Container): columns = self.frum.get_columns(table=self.frum.name) else: columns = [] query_path = coalesce(self.frum.query_path, ".") vars = query_get_all_vars(self, exclude_where=True) # WE WILL EXCLUDE where VARIABLES for c in columns: if c.name in vars and not query_path.startswith(coalesce(listwrap(c.nested_path)[0], "")): Log.error("This query, with variable {{var_name}} is too deep", var_name=c.name)
def query(path): with CProfiler(): try: with Timer("total duration") as query_timer: preamble_timer = Timer("preamble") with preamble_timer: if flask.request.headers.get("content-length", "") in ["", "0"]: # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK return Response(BLANK, status=400, headers={"Content-Type": "text/html"}) elif int(flask.request.headers["content-length"] ) > QUERY_SIZE_LIMIT: Log.error("Query is too large") request_body = flask.request.get_data().strip() text = convert.utf82unicode(request_body) text = replace_vars(text, flask.request.args) data = convert.json2value(text) record_request(flask.request, data, None, None) if data.meta.testing: _test_mode_wait(data) translate_timer = Timer("translate") with translate_timer: if data.sql: data = parse_sql(data.sql) frum = wrap_from(data['from']) result = jx.run(data, frum=frum) if isinstance( result, Container ): #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers result = result.format(data.format) save_timer = Timer("save") with save_timer: if data.meta.save: try: result.meta.saved_as = save_query.query_finder.save( data) except Exception, e: Log.warning("Unexpected save problem", cause=e) result.meta.timing.preamble = Math.round( preamble_timer.duration.seconds, digits=4) result.meta.timing.translate = Math.round( translate_timer.duration.seconds, digits=4) result.meta.timing.save = Math.round( save_timer.duration.seconds, digits=4) result.meta.timing.total = "{{TOTAL_TIME}}" # TIMING PLACEHOLDER with Timer("jsonification") as json_timer: response_data = convert.unicode2utf8( convert.value2json(result)) with Timer("post timer"): # IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF. # WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END timing_replacement = b'"total": ' + str(Math.round(query_timer.duration.seconds, digits=4)) +\ b', "jsonification": ' + str(Math.round(json_timer.duration.seconds, digits=4)) response_data = response_data.replace( b'"total": "{{TOTAL_TIME}}"', timing_replacement) Log.note("Response is {{num}} bytes in {{duration}}", num=len(response_data), duration=query_timer.duration) return Response( response_data, status=200, headers={"Content-Type": result.meta.content_type}) except Exception, e: e = Except.wrap(e) return _send_error(query_timer, request_body, e)
def __init__(self, query, schema=None): """ NORMALIZE QUERY SO IT CAN STILL BE JSON """ if isinstance(query, Query): return object.__init__(self) query = wrap(query) max_depth = 1 self.format = query.format self.frum = wrap_from(query["from"], schema=schema) select = query.select if isinstance(select, list): names = set() new_select = [] for s in select: ns = _normalize_select(s, schema=schema) if ns.name in names: Log.error("two select have the same name") names.add(ns.name) new_select.append(unwrap(ns)) self.select = wrap(new_select) elif select: self.select = _normalize_select(select, schema=schema) else: if query.edges or query.groupby: self.select = {"name": "count", "value": ".", "aggregate": "count"} else: self.select = {"name": "__all__", "value": "*", "aggregate": "none"} if query.groupby and query.edges: Log.error("You can not use both the `groupby` and `edges` clauses in the same query!") elif query.edges: self.edges = _normalize_edges(query.edges, schema=schema) self.groupby = None elif query.groupby: self.edges = None self.groupby = _normalize_groupby(query.groupby, schema=schema) else: self.edges = [] self.groupby = None self.where = _normalize_where(query.where, schema=schema) self.window = [_normalize_window(w) for w in listwrap(query.window)] self.sort = _normalize_sort(query.sort) self.limit = coalesce(query.limit, DEFAULT_LIMIT) if not Math.is_integer(self.limit) or self.limit < 0: Log.error("Expecting limit >= 0") self.isLean = query.isLean # DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN # THE from SOURCE IS. # TODO: IGNORE REACHING INTO THE NON-NESTED TYPES if isinstance(self.frum, list): if not qb: _late_import() columns = qb.get_columns(self.frum) elif isinstance(self.frum, Container): columns = self.frum.get_columns() else: columns = [] vars = get_all_vars(self) for c in columns: if c.name in vars and c.depth: Log.error("This query, with variable {{var_name}} looks too deep", )