def get_raw_json(path):
active_data_timer = Timer("total duration")
body = flask.request.get_data()
try:
with active_data_timer:
args = scrub_args(flask.request.args)
limit = args.limit if args.limit else 10
args.limit = None
frum = find_container(path, after=None)
result = jx.run(
{
"from": path,
"where": {
"eq": args
},
"limit": limit,
"format": "list"
}, frum)
if isinstance(
result, Container
): # TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format("list")
result.meta.active_data_response_time = active_data_timer.duration
response_data = value2json(result.data, pretty=True).encode('utf8')
Log.note("Response is {{num}} bytes", num=len(response_data))
return Response(response_data, status=200)
except Exception as e:
e = Except.wrap(e)
return send_error(active_data_timer, body, e)
def sql_query(path):
with RegisterThread():
query_timer = Timer("total duration")
request_body = None
try:
with query_timer:
preamble_timer = Timer("preamble", silent=True)
with preamble_timer:
if flask.request.headers.get("content-length",
"") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(BLANK,
status=400,
headers={"Content-Type": "text/html"})
elif int(flask.request.headers["content-length"]
) > QUERY_SIZE_LIMIT:
Log.error("Query is too large")
request_body = flask.request.get_data().strip()
text = utf82unicode(request_body)
data = json2value(text)
record_request(flask.request, data, None, None)
translate_timer = Timer("translate", silent=True)
with translate_timer:
if not data.sql:
Log.error("Expecting a `sql` parameter")
jx_query = parse_sql(data.sql)
frum = find_container(jx_query['from'])
if data.meta.testing:
test_mode_wait(jx_query)
result = jx.run(jx_query, container=frum)
if isinstance(
result, Container
): # TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format(jx_query.format)
result.meta.jx_query = jx_query
save_timer = Timer("save")
with save_timer:
if data.meta.save:
try:
result.meta.saved_as = save_query.query_finder.save(
data)
except Exception as e:
Log.warning("Unexpected save problem", cause=e)
result.meta.timing.preamble = mo_math.round(
preamble_timer.duration.seconds, digits=4)
result.meta.timing.translate = mo_math.round(
translate_timer.duration.seconds, digits=4)
result.meta.timing.save = mo_math.round(
save_timer.duration.seconds, digits=4)
result.meta.timing.total = "{{TOTAL_TIME}}" # TIMING PLACEHOLDER
with Timer("jsonification", silent=True) as json_timer:
response_data = unicode2utf8(value2json(result))
with Timer("post timer", silent=True):
# IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.
# WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
timing_replacement = b'"total": ' + str(mo_math.round(query_timer.duration.seconds, digits=4)) +\
b', "jsonification": ' + str(mo_math.round(json_timer.duration.seconds, digits=4))
response_data = response_data.replace(
b'"total":"{{TOTAL_TIME}}"', timing_replacement)
Log.note("Response is {{num}} bytes in {{duration}}",
num=len(response_data),
duration=query_timer.duration)
return Response(
response_data,
status=200,
headers={"Content-Type": result.meta.content_type})
except Exception as e:
e = Except.wrap(e)
return send_error(query_timer, request_body, e)
def jx_query(path):
try:
with Timer("total duration", verbose=DEBUG) as query_timer:
preamble_timer = Timer("preamble", silent=True)
with preamble_timer:
if flask.request.headers.get("content-length", "") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(
BLANK,
status=400,
headers={
"Content-Type": "text/html"
}
)
elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT:
Log.error(QUERY_TOO_LARGE)
request_body = flask.request.get_data().strip()
text = request_body.decode('utf8')
data = json2value(text)
record_request(flask.request, data, None, None)
if data.meta.testing:
test_mode_wait(data, MAIN_THREAD.please_stop)
find_table_timer = Timer("find container", verbose=DEBUG)
with find_table_timer:
frum = find_container(data['from'], after=None)
translate_timer = Timer("translate", verbose=DEBUG)
with translate_timer:
result = jx.run(data, container=frum)
if isinstance(result, Container): # TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format(data.format)
save_timer = Timer("save", verbose=DEBUG)
with save_timer:
if data.meta.save:
try:
result.meta.saved_as = save_query.query_finder.save(data)
except Exception as e:
Log.warning("Unexpected save problem", cause=e)
result.meta.timing.find_table = mo_math.round(find_table_timer.duration.seconds, digits=4)
result.meta.timing.preamble = mo_math.round(preamble_timer.duration.seconds, digits=4)
result.meta.timing.translate = mo_math.round(translate_timer.duration.seconds, digits=4)
result.meta.timing.save = mo_math.round(save_timer.duration.seconds, digits=4)
result.meta.timing.total = "{{TOTAL_TIME}}" # TIMING PLACEHOLDER
with Timer("jsonification", verbose=DEBUG) as json_timer:
response_data = value2json(result).encode('utf8')
with Timer("post timer", verbose=DEBUG):
# IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.
# WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
timing_replacement = (
b'"total":' + binary_type(mo_math.round(query_timer.duration.seconds, digits=4)) +
b', "jsonification":' + binary_type(mo_math.round(json_timer.duration.seconds, digits=4))
)
response_data = response_data.replace(b'"total":"{{TOTAL_TIME}}"', timing_replacement)
Log.note("Response is {{num}} bytes in {{duration}}", num=len(response_data), duration=query_timer.duration)
return Response(
response_data,
status=200,
headers={
"Content-Type": result.meta.content_type
}
)
except Exception as e:
e = Except.wrap(e)
return send_error(query_timer, request_body, e)
def jx_query(path):
with CProfiler():
try:
with Timer("total duration") as query_timer:
preamble_timer = Timer("preamble")
with preamble_timer:
if flask.request.headers.get("content-length",
"") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(BLANK,
status=400,
headers={"Content-Type": "text/html"})
elif int(flask.request.headers["content-length"]
) > QUERY_SIZE_LIMIT:
Log.error("Query is too large")
request_body = flask.request.get_data().strip()
text = convert.utf82unicode(request_body)
data = convert.json2value(text)
record_request(flask.request, data, None, None)
if data.meta.testing:
test_mode_wait(data)
translate_timer = Timer("translate")
with translate_timer:
frum = wrap_from(data['from'])
if any(g == "result.test" or (isinstance(g, Mapping) and
g['value'] == "result.test")
for g in listwrap(data.groupby) +
listwrap(data.edges)):
allowed_query(value2json(data))
result = jx.run(data, frum=frum)
if isinstance(
result, Container
): #TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
result = result.format(data.format)
save_timer = Timer("save")
with save_timer:
if data.meta.save:
try:
result.meta.saved_as = save_query.query_finder.save(
data)
except Exception, e:
Log.warning("Unexpected save problem", cause=e)
result.meta.timing.preamble = Math.round(
preamble_timer.duration.seconds, digits=4)
result.meta.timing.translate = Math.round(
translate_timer.duration.seconds, digits=4)
result.meta.timing.save = Math.round(
save_timer.duration.seconds, digits=4)
result.meta.timing.total = "{{TOTAL_TIME}}" # TIMING PLACEHOLDER
with Timer("jsonification") as json_timer:
response_data = convert.unicode2utf8(
convert.value2json(result))
with Timer("post timer"):
# IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.
# WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
timing_replacement = b'"total": ' + str(Math.round(query_timer.duration.seconds, digits=4)) +\
b', "jsonification": ' + str(Math.round(json_timer.duration.seconds, digits=4))
response_data = response_data.replace(
b'"total": "{{TOTAL_TIME}}"', timing_replacement)
Log.note("Response is {{num}} bytes in {{duration}}",
num=len(response_data),
duration=query_timer.duration)
return Response(
response_data,
status=200,
headers={"Content-Type": result.meta.content_type})
except Exception, e:
e = Except.wrap(e)
return send_error(query_timer, request_body, e)
# WE CHEAT BY DOING A (HOPEFULLY FAST) STRING REPLACEMENT AT THE VERY END
timing_replacement = b'"total": ' + str(Math.round(query_timer.duration.seconds, digits=4)) +\
b', "jsonification": ' + str(Math.round(json_timer.duration.seconds, digits=4))
response_data = response_data.replace(b'"total": "{{TOTAL_TIME}}"', timing_replacement)
Log.note("Response is {{num}} bytes in {{duration}}", num=len(response_data), duration=query_timer.duration)
return Response(
response_data,
status=200,
headers={
"Content-Type": result.meta.content_type
}
)
except Exception as e:
e = Except.wrap(e)
return send_error(query_timer, request_body, e)
KNOWN_SQL_AGGREGATES = {"sum", "count", "avg"}
def parse_sql(sql):
query = wrap(moz_sql_parser.parse(sql))
# PULL OUT THE AGGREGATES
for s in listwrap(query.select):
val = s.value
# LOOK FOR GROUPBY COLUMN IN SELECT CLAUSE, REMOVE DUPLICATION
for g in listwrap(query.groupby):
try:
assertAlmostEqual(g.value, val, "")
g.name = s.name