def compserver(payload, serial):
ns = payload.get('namespace', dict())
compute_kwargs = payload.get('compute_kwargs') or {}
odo_kwargs = payload.get('odo_kwargs') or {}
dataset = _get_data()
ns[':leaf'] = symbol('leaf', discover(dataset))
expr = from_tree(payload['expr'], namespace=ns)
assert len(expr._leaves()) == 1
leaf = expr._leaves()[0]
try:
result = compute(expr, {leaf: dataset}, **compute_kwargs)
if iscollection(expr.dshape):
result = odo(result, list, **odo_kwargs)
elif isscalar(expr.dshape):
result = coerce_scalar(result, str(expr.dshape))
except NotImplementedError as e:
# 501: Not Implemented
return ("Computation not supported:\n%s" % e, 501)
except Exception as e:
# 500: Internal Server Error
return (
"Computation failed with message:\n%s: %s" % (type(e).__name__, e),
500,
)
return serial.dumps({
'datashape': pprint(expr.dshape, width=0),
'data': result,
'names': expr.fields
})
def compserver(payload, serial):
ns = payload.get('namespace', dict())
compute_kwargs = payload.get('compute_kwargs') or {}
odo_kwargs = payload.get('odo_kwargs') or {}
dataset = _get_data()
ns[':leaf'] = symbol('leaf', discover(dataset))
expr = from_tree(payload['expr'], namespace=ns)
assert len(expr._leaves()) == 1
leaf = expr._leaves()[0]
try:
result = compute(expr, {leaf: dataset}, **compute_kwargs)
if iscollection(expr.dshape):
result = odo(result, list, **odo_kwargs)
elif isscalar(expr.dshape):
result = coerce_scalar(result, str(expr.dshape))
except NotImplementedError as e:
# 501: Not Implemented
return ("Computation not supported:\n%s" % e, 501)
except Exception as e:
# 500: Internal Server Error
return (
"Computation failed with message:\n%s: %s" % (type(e).__name__, e),
500,
)
return serial.dumps({
'datashape': pprint(expr.dshape, width=0),
'data': result,
'names': expr.fields
})
def compserver(payload, serial):
(allow_profiler,
default_profiler_output,
profile_by_default) = _get_profiler_info()
requested_profiler_output = payload.get('profiler_output',
default_profiler_output)
profile = payload.get('profile')
profiling = (allow_profiler and
(profile or (profile_by_default and requested_profiler_output)))
if profile and not allow_profiler:
return ('profiling is disabled on this server', RC.FORBIDDEN)
with ExitStack() as response_construction_context_stack:
if profiling:
from cProfile import Profile
if (default_profiler_output == ':response' and
requested_profiler_output != ':response'):
# writing to the local filesystem is disabled
return ("local filepaths are disabled on this server, only"
" ':response' is allowed for the 'profiler_output' field",
RC.FORBIDDEN)
profiler_output = requested_profiler_output
profiler = Profile()
profiler.enable()
# ensure that we stop profiling in the case of an exception
response_construction_context_stack.callback(profiler.disable)
expr = '<failed to parse expr>'
@response_construction_context_stack.callback
def log_time(start=time()):
flask.current_app.logger.info('compute expr: %s\ntotal time (s): %.3f',
expr,
time() - start)
ns = payload.get('namespace', {})
compute_kwargs = payload.get('compute_kwargs') or {}
odo_kwargs = payload.get('odo_kwargs') or {}
dataset = _get_data()
ns[':leaf'] = symbol('leaf', discover(dataset))
expr = from_tree(payload['expr'], namespace=ns)
assert len(expr._leaves()) == 1
leaf = expr._leaves()[0]
try:
result = serial.materialize(compute(expr,
{leaf: dataset},
**compute_kwargs),
expr.dshape,
odo_kwargs)
except NotImplementedError as e:
return ("Computation not supported:\n%s" % e, RC.NOT_IMPLEMENTED)
except Exception as e:
return ("Computation failed with message:\n%s: %s" % (type(e).__name__, e),
RC.INTERNAL_SERVER_ERROR)
response = {'datashape': pprint(expr.dshape, width=0),
'data': serial.data_dumps(result),
'names': expr.fields}
if profiling:
import marshal
from pstats import Stats
if profiler_output == ':response':
from pandas.compat import BytesIO
file = BytesIO()
else:
file = open(_prof_path(profiler_output, expr), 'wb')
with file:
# Use marshal to dump the stats data to the given file.
# This is taken from cProfile which unfortunately does not have
# an api that allows us to pass the file object directly, only
# a file path.
marshal.dump(Stats(profiler).stats, file)
if profiler_output == ':response':
response['profiler_output'] = {'__!bytes': file.getvalue()}
return serial.dumps(response)
def shape():
return pprint(discover(_get_data()), width=0)
def compserver(payload, serial):
app = flask.current_app
(allow_profiler,
default_profiler_output,
profile_by_default) = _get_profiler_info()
requested_profiler_output = payload.get(u'profiler_output',
default_profiler_output)
profile = payload.get(u'profile')
profiling = (allow_profiler and
(profile or (profile_by_default and requested_profiler_output)))
if profile and not allow_profiler:
return ('profiling is disabled on this server', RC.FORBIDDEN)
with ExitStack() as response_construction_context_stack:
if profiling:
from cProfile import Profile
if (default_profiler_output == ':response' and
requested_profiler_output != ':response'):
# writing to the local filesystem is disabled
return ("local filepaths are disabled on this server, only"
" ':response' is allowed for the 'profiler_output' field",
RC.FORBIDDEN)
profiler_output = requested_profiler_output
profiler = Profile()
profiler.enable()
# ensure that we stop profiling in the case of an exception
response_construction_context_stack.callback(profiler.disable)
expr = '<failed to parse expr>'
@response_construction_context_stack.callback
def log_time(start=time()):
app.logger.info('compute expr: %s\ntotal time (s): %.3f',
expr,
time() - start)
ns = payload.get(u'namespace', {})
compute_kwargs = payload.get(u'compute_kwargs') or {}
odo_kwargs = payload.get(u'odo_kwargs') or {}
dataset = _get_data()
ns[':leaf'] = symbol('leaf', discover(dataset))
expr = from_tree(payload[u'expr'], namespace=ns)
assert len(expr._leaves()) == 1
leaf = expr._leaves()[0]
try:
formatter = getattr(flask.current_app, 'log_exception_formatter',
_default_log_exception_formatter)
result = serial.materialize(compute(expr,
{leaf: dataset},
**compute_kwargs),
expr.dshape,
odo_kwargs)
except NotImplementedError as e:
# Note: `sys.exc_info()[2]` holds the current traceback, for
# Python 2 / 3 compatibility. It's important not to store a local
# reference to it.
formatted_tb = formatter(sys.exc_info()[2])
error_msg = "Computation not supported:\n%s\n%s" % (e, formatted_tb)
app.logger.error(error_msg)
return (error_msg, RC.NOT_IMPLEMENTED)
except Exception as e:
formatted_tb = formatter(sys.exc_info()[2])
error_msg = "Computation failed with message:\n%s: %s\n%s" % (type(e).__name__, e, formatted_tb)
app.logger.error(error_msg)
return (error_msg, RC.INTERNAL_SERVER_ERROR)
response = {u'datashape': pprint(expr.dshape, width=0),
u'data': serial.data_dumps(result),
u'names': expr.fields}
if profiling:
import marshal
from pstats import Stats
if profiler_output == ':response':
from pandas.compat import BytesIO
file = BytesIO()
else:
file = open(_prof_path(profiler_output, expr), 'wb')
with file:
# Use marshal to dump the stats data to the given file.
# This is taken from cProfile which unfortunately does not have
# an api that allows us to pass the file object directly, only
# a file path.
marshal.dump(Stats(profiler).stats, file)
if profiler_output == ':response':
response[u'profiler_output'] = {'__!bytes': file.getvalue()}
return serial.dumps(response)
def shape():
return pprint(discover(_get_data()), width=0)
def dshape_print(dshape):
return ds.pprint(dshape, 1000000)
def sanitized_dshape(dshape, width=50):
pretty_dshape = datashape.pprint(dshape, width=width).replace('\n', '')
if len(pretty_dshape) > width:
pretty_dshape = "{}...".format(pretty_dshape[:width])
return pretty_dshape
def compserver(payload, serial):
expected_keys = {u'namespace',
u'odo_kwargs',
u'compute_kwargs',
u'expr',
u'profile',
u'profiler_output'}
if not set(payload.keys()) < expected_keys:
return ('unexpected keys in payload: %r' % sorted(set(payload.keys()) -
expected_keys),
RC.BAD_REQUEST)
app = flask.current_app
(allow_profiler,
default_profiler_output,
profile_by_default) = _get_profiler_info()
requested_profiler_output = payload.get(u'profiler_output',
default_profiler_output)
profile = payload.get(u'profile')
profiling = (allow_profiler and
(profile or (profile_by_default and requested_profiler_output)))
if profile and not allow_profiler:
return ('profiling is disabled on this server', RC.FORBIDDEN)
with ExitStack() as response_construction_context_stack:
if profiling:
from cProfile import Profile
if (default_profiler_output == ':response' and
requested_profiler_output != ':response'):
# writing to the local filesystem is disabled
return ("local filepaths are disabled on this server, only"
" ':response' is allowed for the 'profiler_output' field",
RC.FORBIDDEN)
profiler_output = requested_profiler_output
profiler = Profile()
profiler.enable()
# ensure that we stop profiling in the case of an exception
response_construction_context_stack.callback(profiler.disable)
expr = '<failed to parse expr>'
@response_construction_context_stack.callback
def log_time(start=time()):
app.logger.info('compute expr: %s\ntotal time (s): %.3f',
expr,
time() - start)
ns = payload.get(u'namespace', {})
compute_kwargs = payload.get(u'compute_kwargs') or {}
odo_kwargs = payload.get(u'odo_kwargs') or {}
dataset = _get_data()
ns[':leaf'] = symbol('leaf', discover(dataset))
expr = from_tree(payload[u'expr'], namespace=ns)
if len(expr._leaves()) != 1:
return ('too many leaves, expected 1 got %d' % len(expr._leaves()),
RC.BAD_REQUEST)
leaf = expr._leaves()[0]
formatter = getattr(flask.current_app, 'log_exception_formatter',
_default_log_exception_formatter)
try:
result = serial.materialize(compute(expr,
{leaf: dataset},
**compute_kwargs),
expr.dshape,
odo_kwargs)
except NotImplementedError as e:
# Note: `sys.exc_info()[2]` holds the current traceback, for
# Python 2 / 3 compatibility. It's important not to store a local
# reference to it.
formatted_tb = formatter(sys.exc_info()[2])
error_msg = "Computation not supported:\n%s\n%s" % (e, formatted_tb)
app.logger.error(error_msg)
return (error_msg, RC.NOT_IMPLEMENTED)
except Exception as e:
formatted_tb = formatter(sys.exc_info()[2])
error_msg = "Computation failed with message:\n%s: %s\n%s" % (type(e).__name__, e, formatted_tb)
app.logger.error(error_msg)
return (error_msg, RC.INTERNAL_SERVER_ERROR)
response = {u'datashape': pprint(expr.dshape, width=0),
u'data': serial.data_dumps(result),
u'names': expr.fields}
if profiling:
import marshal
from pstats import Stats
if profiler_output == ':response':
from pandas.compat import BytesIO
file = BytesIO()
else:
file = open(_prof_path(profiler_output, expr), 'wb')
with file:
# Use marshal to dump the stats data to the given file.
# This is taken from cProfile which unfortunately does not have
# an api that allows us to pass the file object directly, only
# a file path.
marshal.dump(Stats(profiler).stats, file)
if profiler_output == ':response':
response[u'profiler_output'] = {'__!bytes': file.getvalue()}
return serial.dumps(response)
def dshape_print(dshape: Union[ds.Mono, str]) -> str:
return ds.pprint(dshape, 1000000)