Example #1
0
def compserver(payload, serial):
    ns = payload.get('namespace', dict())
    compute_kwargs = payload.get('compute_kwargs') or {}
    odo_kwargs = payload.get('odo_kwargs') or {}
    dataset = _get_data()
    ns[':leaf'] = symbol('leaf', discover(dataset))

    expr = from_tree(payload['expr'], namespace=ns)
    assert len(expr._leaves()) == 1
    leaf = expr._leaves()[0]

    try:
        result = compute(expr, {leaf: dataset}, **compute_kwargs)

        if iscollection(expr.dshape):
            result = odo(result, list, **odo_kwargs)
        elif isscalar(expr.dshape):
            result = coerce_scalar(result, str(expr.dshape))
    except NotImplementedError as e:
        # 501: Not Implemented
        return ("Computation not supported:\n%s" % e, 501)
    except Exception as e:
        # 500: Internal Server Error
        return (
            "Computation failed with message:\n%s: %s" % (type(e).__name__, e),
            500,
        )

    return serial.dumps({
        'datashape': pprint(expr.dshape, width=0),
        'data': result,
        'names': expr.fields
    })
Example #2
0
def compserver(payload, serial):
    ns = payload.get('namespace', dict())
    compute_kwargs = payload.get('compute_kwargs') or {}
    odo_kwargs = payload.get('odo_kwargs') or {}
    dataset = _get_data()
    ns[':leaf'] = symbol('leaf', discover(dataset))

    expr = from_tree(payload['expr'], namespace=ns)
    assert len(expr._leaves()) == 1
    leaf = expr._leaves()[0]

    try:
        result = compute(expr, {leaf: dataset}, **compute_kwargs)

        if iscollection(expr.dshape):
            result = odo(result, list, **odo_kwargs)
        elif isscalar(expr.dshape):
            result = coerce_scalar(result, str(expr.dshape))
    except NotImplementedError as e:
        # 501: Not Implemented
        return ("Computation not supported:\n%s" % e, 501)
    except Exception as e:
        # 500: Internal Server Error
        return (
            "Computation failed with message:\n%s: %s" % (type(e).__name__, e),
            500,
        )

    return serial.dumps({
        'datashape': pprint(expr.dshape, width=0),
        'data': result,
        'names': expr.fields
    })
Example #3
0
def compserver(payload, serial):
    (allow_profiler,
     default_profiler_output,
     profile_by_default) = _get_profiler_info()
    requested_profiler_output = payload.get('profiler_output',
                                            default_profiler_output)
    profile = payload.get('profile')
    profiling = (allow_profiler and
                 (profile or (profile_by_default and requested_profiler_output)))
    if profile and not allow_profiler:
        return ('profiling is disabled on this server', RC.FORBIDDEN)

    with ExitStack() as response_construction_context_stack:
        if profiling:
            from cProfile import Profile

            if (default_profiler_output == ':response' and
                    requested_profiler_output != ':response'):
                # writing to the local filesystem is disabled
                return ("local filepaths are disabled on this server, only"
                        " ':response' is allowed for the 'profiler_output' field",
                        RC.FORBIDDEN)

            profiler_output = requested_profiler_output
            profiler = Profile()
            profiler.enable()
            # ensure that we stop profiling in the case of an exception
            response_construction_context_stack.callback(profiler.disable)

        expr = '<failed to parse expr>'

        @response_construction_context_stack.callback
        def log_time(start=time()):
            flask.current_app.logger.info('compute expr: %s\ntotal time (s): %.3f',
                                          expr,
                                          time() - start)

        ns = payload.get('namespace', {})
        compute_kwargs = payload.get('compute_kwargs') or {}
        odo_kwargs = payload.get('odo_kwargs') or {}
        dataset = _get_data()
        ns[':leaf'] = symbol('leaf', discover(dataset))

        expr = from_tree(payload['expr'], namespace=ns)
        assert len(expr._leaves()) == 1
        leaf = expr._leaves()[0]

        try:
            result = serial.materialize(compute(expr,
                                                {leaf: dataset},
                                                **compute_kwargs),
                                        expr.dshape,
                                        odo_kwargs)
        except NotImplementedError as e:
            return ("Computation not supported:\n%s" % e, RC.NOT_IMPLEMENTED)
        except Exception as e:
            return ("Computation failed with message:\n%s: %s" % (type(e).__name__, e),
                    RC.INTERNAL_SERVER_ERROR)

        response = {'datashape': pprint(expr.dshape, width=0),
                    'data': serial.data_dumps(result),
                    'names': expr.fields}

    if profiling:
        import marshal
        from pstats import Stats

        if profiler_output == ':response':
            from pandas.compat import BytesIO
            file = BytesIO()
        else:
            file = open(_prof_path(profiler_output, expr), 'wb')

        with file:
            # Use marshal to dump the stats data to the given file.
            # This is taken from cProfile which unfortunately does not have
            # an api that allows us to pass the file object directly, only
            # a file path.
            marshal.dump(Stats(profiler).stats, file)
            if profiler_output == ':response':
                response['profiler_output'] = {'__!bytes': file.getvalue()}

    return serial.dumps(response)
Example #4
0
def shape():
    return pprint(discover(_get_data()), width=0)
Example #5
0
def compserver(payload, serial):
    app = flask.current_app
    (allow_profiler,
     default_profiler_output,
     profile_by_default) = _get_profiler_info()
    requested_profiler_output = payload.get(u'profiler_output',
                                            default_profiler_output)
    profile = payload.get(u'profile')
    profiling = (allow_profiler and
                 (profile or (profile_by_default and requested_profiler_output)))
    if profile and not allow_profiler:
        return ('profiling is disabled on this server', RC.FORBIDDEN)

    with ExitStack() as response_construction_context_stack:
        if profiling:
            from cProfile import Profile

            if (default_profiler_output == ':response' and
                    requested_profiler_output != ':response'):
                # writing to the local filesystem is disabled
                return ("local filepaths are disabled on this server, only"
                        " ':response' is allowed for the 'profiler_output' field",
                        RC.FORBIDDEN)

            profiler_output = requested_profiler_output
            profiler = Profile()
            profiler.enable()
            # ensure that we stop profiling in the case of an exception
            response_construction_context_stack.callback(profiler.disable)

        expr = '<failed to parse expr>'

        @response_construction_context_stack.callback
        def log_time(start=time()):
            app.logger.info('compute expr: %s\ntotal time (s): %.3f',
                            expr,
                            time() - start)

        ns = payload.get(u'namespace', {})
        compute_kwargs = payload.get(u'compute_kwargs') or {}
        odo_kwargs = payload.get(u'odo_kwargs') or {}
        dataset = _get_data()
        ns[':leaf'] = symbol('leaf', discover(dataset))

        expr = from_tree(payload[u'expr'], namespace=ns)
        assert len(expr._leaves()) == 1
        leaf = expr._leaves()[0]

        try:
            formatter = getattr(flask.current_app, 'log_exception_formatter',
                                _default_log_exception_formatter)
            result = serial.materialize(compute(expr,
                                                {leaf: dataset},
                                                **compute_kwargs),
                                        expr.dshape,
                                        odo_kwargs)
        except NotImplementedError as e:
            # Note: `sys.exc_info()[2]` holds the current traceback, for
            # Python 2 / 3 compatibility. It's important not to store a local
            # reference to it.
            formatted_tb = formatter(sys.exc_info()[2])
            error_msg = "Computation not supported:\n%s\n%s" % (e, formatted_tb)
            app.logger.error(error_msg)
            return (error_msg, RC.NOT_IMPLEMENTED)
        except Exception as e:
            formatted_tb = formatter(sys.exc_info()[2])
            error_msg = "Computation failed with message:\n%s: %s\n%s" % (type(e).__name__, e, formatted_tb)
            app.logger.error(error_msg)
            return (error_msg, RC.INTERNAL_SERVER_ERROR)

        response = {u'datashape': pprint(expr.dshape, width=0),
                    u'data': serial.data_dumps(result),
                    u'names': expr.fields}

    if profiling:
        import marshal
        from pstats import Stats

        if profiler_output == ':response':
            from pandas.compat import BytesIO
            file = BytesIO()
        else:
            file = open(_prof_path(profiler_output, expr), 'wb')

        with file:
            # Use marshal to dump the stats data to the given file.
            # This is taken from cProfile which unfortunately does not have
            # an api that allows us to pass the file object directly, only
            # a file path.
            marshal.dump(Stats(profiler).stats, file)
            if profiler_output == ':response':
                response[u'profiler_output'] = {'__!bytes': file.getvalue()}

    return serial.dumps(response)
Example #6
0
def shape():
    return pprint(discover(_get_data()), width=0)
Example #7
0
def dshape_print(dshape):
    return ds.pprint(dshape, 1000000)
Example #8
0
def sanitized_dshape(dshape, width=50):
    pretty_dshape = datashape.pprint(dshape, width=width).replace('\n', '')
    if len(pretty_dshape) > width:
        pretty_dshape = "{}...".format(pretty_dshape[:width])
    return pretty_dshape
Example #9
0
def compserver(payload, serial):
    expected_keys = {u'namespace',
                     u'odo_kwargs',
                     u'compute_kwargs',
                     u'expr',
                     u'profile',
                     u'profiler_output'}
    if not set(payload.keys()) < expected_keys:
        return ('unexpected keys in payload: %r' % sorted(set(payload.keys()) -
                                                          expected_keys),
                RC.BAD_REQUEST)

    app = flask.current_app
    (allow_profiler,
     default_profiler_output,
     profile_by_default) = _get_profiler_info()
    requested_profiler_output = payload.get(u'profiler_output',
                                            default_profiler_output)
    profile = payload.get(u'profile')
    profiling = (allow_profiler and
                 (profile or (profile_by_default and requested_profiler_output)))
    if profile and not allow_profiler:
        return ('profiling is disabled on this server', RC.FORBIDDEN)

    with ExitStack() as response_construction_context_stack:
        if profiling:
            from cProfile import Profile

            if (default_profiler_output == ':response' and
                    requested_profiler_output != ':response'):
                # writing to the local filesystem is disabled
                return ("local filepaths are disabled on this server, only"
                        " ':response' is allowed for the 'profiler_output' field",
                        RC.FORBIDDEN)

            profiler_output = requested_profiler_output
            profiler = Profile()
            profiler.enable()
            # ensure that we stop profiling in the case of an exception
            response_construction_context_stack.callback(profiler.disable)

        expr = '<failed to parse expr>'

        @response_construction_context_stack.callback
        def log_time(start=time()):
            app.logger.info('compute expr: %s\ntotal time (s): %.3f',
                            expr,
                            time() - start)

        ns = payload.get(u'namespace', {})
        compute_kwargs = payload.get(u'compute_kwargs') or {}
        odo_kwargs = payload.get(u'odo_kwargs') or {}
        dataset = _get_data()
        ns[':leaf'] = symbol('leaf', discover(dataset))

        expr = from_tree(payload[u'expr'], namespace=ns)

        if len(expr._leaves()) != 1:
            return ('too many leaves, expected 1 got %d' % len(expr._leaves()),
                    RC.BAD_REQUEST)

        leaf = expr._leaves()[0]

        formatter = getattr(flask.current_app, 'log_exception_formatter',
                            _default_log_exception_formatter)
        try:
            result = serial.materialize(compute(expr,
                                                {leaf: dataset},
                                                **compute_kwargs),
                                        expr.dshape,
                                        odo_kwargs)
        except NotImplementedError as e:
            # Note: `sys.exc_info()[2]` holds the current traceback, for
            # Python 2 / 3 compatibility. It's important not to store a local
            # reference to it.
            formatted_tb = formatter(sys.exc_info()[2])
            error_msg = "Computation not supported:\n%s\n%s" % (e, formatted_tb)
            app.logger.error(error_msg)
            return (error_msg, RC.NOT_IMPLEMENTED)
        except Exception as e:
            formatted_tb = formatter(sys.exc_info()[2])
            error_msg = "Computation failed with message:\n%s: %s\n%s" % (type(e).__name__, e, formatted_tb)
            app.logger.error(error_msg)
            return (error_msg, RC.INTERNAL_SERVER_ERROR)

        response = {u'datashape': pprint(expr.dshape, width=0),
                    u'data': serial.data_dumps(result),
                    u'names': expr.fields}

    if profiling:
        import marshal
        from pstats import Stats

        if profiler_output == ':response':
            from pandas.compat import BytesIO
            file = BytesIO()
        else:
            file = open(_prof_path(profiler_output, expr), 'wb')

        with file:
            # Use marshal to dump the stats data to the given file.
            # This is taken from cProfile which unfortunately does not have
            # an api that allows us to pass the file object directly, only
            # a file path.
            marshal.dump(Stats(profiler).stats, file)
            if profiler_output == ':response':
                response[u'profiler_output'] = {'__!bytes': file.getvalue()}

    return serial.dumps(response)
Example #10
0
def dshape_print(dshape: Union[ds.Mono, str]) -> str:
    return ds.pprint(dshape, 1000000)