def get_raw_json(path):
active_data_timer = Timer("total duration")
body = flask.request.get_data()
try:
with active_data_timer:
args = wrap(Data(**flask.request.args))
limit = args.limit if args.limit else 10
args.limit = None
frum = wrap_from(path)
result = jx.run(
{
"from": path,
"where": {
"eq": args
},
"limit": limit,
"format": "list"
}, frum)
if isinstance(
result, Container
): #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format("list")
result.meta.active_data_response_time = active_data_timer.duration
response_data = convert.unicode2utf8(
convert.value2json(result.data, pretty=True))
Log.note("Response is {{num}} bytes", num=len(response_data))
return Response(response_data, status=200)
except Exception, e:
e = Except.wrap(e)
return _send_error(active_data_timer, body, e)
def wrap(query, schema=None):
"""
NORMALIZE QUERY SO IT CAN STILL BE JSON
"""
if isinstance(query, QueryOp) or query == None:
return query
query = wrap(query)
output = QueryOp("from", None)
output.format = query.format
output.frum = wrap_from(query["from"], schema=schema)
if not schema and isinstance(output.frum, Schema):
schema = output.frum
if not schema and hasattr(output.frum, "schema"):
schema = output.frum.schema
if query.select or isinstance(query.select, (Mapping, list)):
output.select = _normalize_selects(query.select,
query.frum,
schema=schema)
else:
if query.edges or query.groupby:
output.select = Data(name="count",
value=jx_expression("."),
aggregate="count",
default=0)
else:
output.select = _normalize_selects(".", query.frum)
if query.groupby and query.edges:
Log.error(
"You can not use both the `groupby` and `edges` clauses in the same query!"
)
elif query.edges:
output.edges = _normalize_edges(query.edges, schema=schema)
output.groupby = Null
elif query.groupby:
output.edges = Null
output.groupby = _normalize_groupby(query.groupby, schema=schema)
else:
output.edges = Null
output.groupby = Null
output.where = _normalize_where(query.where, schema=schema)
output.window = [_normalize_window(w) for w in listwrap(query.window)]
output.having = None
output.sort = _normalize_sort(query.sort)
output.limit = Math.min(MAX_LIMIT, coalesce(query.limit,
DEFAULT_LIMIT))
if not Math.is_integer(output.limit) or output.limit < 0:
Log.error("Expecting limit >= 0")
output.isLean = query.isLean
return output
def allowed_query(data):
data = json2value(data)
data.edges = None
data.groupby = None
data.select = {"value": "result.test", "aggregate": "cardinality"}
data.format = "list"
frum = wrap_from(data['from'])
result = jx.run(data, frum=frum)
if result.data < 10000:
return True
else:
Log.error("not allowed to groupby result.test")
def wrap(query, schema=None):
"""
NORMALIZE QUERY SO IT CAN STILL BE JSON
"""
if isinstance(query, QueryOp) or query == None:
return query
query = wrap(query)
output = QueryOp("from", None)
output.format = query.format
output.frum = wrap_from(query["from"], schema=schema)
if not schema and isinstance(output.frum, Schema):
schema = output.frum
if not schema and hasattr(output.frum, "schema"):
schema = output.frum.schema
if query.select or isinstance(query.select, (Mapping, list)):
output.select = _normalize_selects(query.select, query.frum, schema=schema)
else:
if query.edges or query.groupby:
output.select = Data(name="count", value=jx_expression("."), aggregate="count", default=0)
else:
output.select = _normalize_selects(".", query.frum)
if query.groupby and query.edges:
Log.error("You can not use both the `groupby` and `edges` clauses in the same query!")
elif query.edges:
output.edges = _normalize_edges(query.edges, schema=schema)
output.groupby = Null
elif query.groupby:
output.edges = Null
output.groupby = _normalize_groupby(query.groupby, schema=schema)
else:
output.edges = Null
output.groupby = Null
output.where = _normalize_where(query.where, schema=schema)
output.window = [_normalize_window(w) for w in listwrap(query.window)]
output.having = None
output.sort = _normalize_sort(query.sort)
output.limit = Math.min(MAX_LIMIT, coalesce(query.limit, DEFAULT_LIMIT))
if not Math.is_integer(output.limit) or output.limit < 0:
Log.error("Expecting limit >= 0")
output.isLean = query.isLean
return output
def __init__(self, query, schema=None):
"""
NORMALIZE QUERY SO IT CAN STILL BE JSON
"""
if isinstance(query, Query) or query == None:
return
object.__init__(self)
query = wrap(query)
self.format = query.format
self.frum = wrap_from(query["from"], schema=schema)
select = query.select
if isinstance(select, list):
names = set()
new_select = []
for s in select:
ns = _normalize_select(s, schema=schema)
if ns.name in names:
Log.error("two select have the same name")
names.add(ns.name)
new_select.append(unwrap(ns))
self.select = wrap(new_select)
elif select:
self.select = _normalize_select(select, schema=schema)
else:
if query.edges or query.groupby:
self.select = Dict(name="count", value=".", aggregate="count")
else:
self.select = Dict(name=".", value=".", aggregate="none")
if query.groupby and query.edges:
Log.error("You can not use both the `groupby` and `edges` clauses in the same query!")
elif query.edges:
self.edges = _normalize_edges(query.edges, schema=schema)
self.groupby = None
elif query.groupby:
self.edges = None
self.groupby = _normalize_groupby(query.groupby, schema=schema)
else:
self.edges = []
self.groupby = None
self.where = _normalize_where(query.where, schema=schema)
self.window = [_normalize_window(w) for w in listwrap(query.window)]
self.having = None
self.sort = _normalize_sort(query.sort)
self.limit = Math.min(MAX_LIMIT, coalesce(query.limit, DEFAULT_LIMIT))
if not Math.is_integer(self.limit) or self.limit < 0:
Log.error("Expecting limit >= 0")
self.isLean = query.isLean
# DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN
# THE from SOURCE IS.
# TODO: IGNORE REACHING INTO THE NON-NESTED TYPES
if isinstance(self.frum, list):
if not qb:
_late_import()
columns = qb.get_columns(self.frum)
elif isinstance(self.frum, Container):
columns = self.frum.get_columns(table=self.frum.name)
else:
columns = []
query_path = coalesce(self.frum.query_path, ".")
vars = query_get_all_vars(self, exclude_where=True) # WE WILL EXCLUDE where VARIABLES
for c in columns:
if c.name in vars and not query_path.startswith(coalesce(listwrap(c.nested_path)[0], "")):
Log.error("This query, with variable {{var_name}} is too deep", var_name=c.name)
def query(path):
with CProfiler():
try:
with Timer("total duration") as query_timer:
preamble_timer = Timer("preamble")
with preamble_timer:
if flask.request.headers.get("content-length",
"") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(BLANK,
status=400,
headers={"Content-Type": "text/html"})
elif int(flask.request.headers["content-length"]
) > QUERY_SIZE_LIMIT:
Log.error("Query is too large")
request_body = flask.request.get_data().strip()
text = convert.utf82unicode(request_body)
text = replace_vars(text, flask.request.args)
data = convert.json2value(text)
record_request(flask.request, data, None, None)
if data.meta.testing:
_test_mode_wait(data)
translate_timer = Timer("translate")
with translate_timer:
if data.sql:
data = parse_sql(data.sql)
frum = wrap_from(data['from'])
result = jx.run(data, frum=frum)
if isinstance(
result, Container
): #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format(data.format)
save_timer = Timer("save")
with save_timer:
if data.meta.save:
try:
result.meta.saved_as = save_query.query_finder.save(
data)
except Exception, e:
Log.warning("Unexpected save problem", cause=e)
result.meta.timing.preamble = Math.round(
preamble_timer.duration.seconds, digits=4)
result.meta.timing.translate = Math.round(
translate_timer.duration.seconds, digits=4)
result.meta.timing.save = Math.round(
save_timer.duration.seconds, digits=4)
result.meta.timing.total = "{{TOTAL_TIME}}" # TIMING PLACEHOLDER
with Timer("jsonification") as json_timer:
response_data = convert.unicode2utf8(
convert.value2json(result))
with Timer("post timer"):
# IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.
# WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
timing_replacement = b'"total": ' + str(Math.round(query_timer.duration.seconds, digits=4)) +\
b', "jsonification": ' + str(Math.round(json_timer.duration.seconds, digits=4))
response_data = response_data.replace(
b'"total": "{{TOTAL_TIME}}"', timing_replacement)
Log.note("Response is {{num}} bytes in {{duration}}",
num=len(response_data),
duration=query_timer.duration)
return Response(
response_data,
status=200,
headers={"Content-Type": result.meta.content_type})
except Exception, e:
e = Except.wrap(e)
return _send_error(query_timer, request_body, e)
def __init__(self, query, schema=None):
"""
NORMALIZE QUERY SO IT CAN STILL BE JSON
"""
if isinstance(query, Query):
return
object.__init__(self)
query = wrap(query)
max_depth = 1
self.format = query.format
self.frum = wrap_from(query["from"], schema=schema)
select = query.select
if isinstance(select, list):
names = set()
new_select = []
for s in select:
ns = _normalize_select(s, schema=schema)
if ns.name in names:
Log.error("two select have the same name")
names.add(ns.name)
new_select.append(unwrap(ns))
self.select = wrap(new_select)
elif select:
self.select = _normalize_select(select, schema=schema)
else:
if query.edges or query.groupby:
self.select = {"name": "count", "value": ".", "aggregate": "count"}
else:
self.select = {"name": "__all__", "value": "*", "aggregate": "none"}
if query.groupby and query.edges:
Log.error("You can not use both the `groupby` and `edges` clauses in the same query!")
elif query.edges:
self.edges = _normalize_edges(query.edges, schema=schema)
self.groupby = None
elif query.groupby:
self.edges = None
self.groupby = _normalize_groupby(query.groupby, schema=schema)
else:
self.edges = []
self.groupby = None
self.where = _normalize_where(query.where, schema=schema)
self.window = [_normalize_window(w) for w in listwrap(query.window)]
self.sort = _normalize_sort(query.sort)
self.limit = coalesce(query.limit, DEFAULT_LIMIT)
if not Math.is_integer(self.limit) or self.limit < 0:
Log.error("Expecting limit >= 0")
self.isLean = query.isLean
# DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN
# THE from SOURCE IS.
# TODO: IGNORE REACHING INTO THE NON-NESTED TYPES
if isinstance(self.frum, list):
if not qb:
_late_import()
columns = qb.get_columns(self.frum)
elif isinstance(self.frum, Container):
columns = self.frum.get_columns()
else:
columns = []
vars = get_all_vars(self)
for c in columns:
if c.name in vars and c.depth:
Log.error("This query, with variable {{var_name}} looks too deep", )
