def _serialize_external_object(self, o):
yield "System", "Python"
if callable(o):
yield "Type", "PythonFunction"
try:
# force tuple to avoid calling this method again on `map`.
yield "Arguments", tuple(map(force_text, first(inspect.getfullargspec(o))))
except TypeError:
# this function can fail with TypeError unsupported callable
pass
else:
yield "Type", "PythonObject"
if hasattr(o, "__name__"):
yield "Command", force_text(o.__name__)
else:
yield "Command", force_text(o.__class__.__name__)
is_module = inspect.ismodule(o)
yield "IsModule", is_module
if not is_module:
module = inspect.getmodule(o)
if module:
yield "Module", force_text(module.__name__)
yield "IsClass", inspect.isclass(o),
yield "IsFunction", inspect.isfunction(o),
yield "IsMethod", inspect.ismethod(o),
yield "IsCallable", callable(o),
def _serialize_external_object(self, o):
yield "System", "Python"
yield "Type", "PythonFunction"
if hasattr(o, '__name__'):
yield "Name", force_text(o.__name__)
else:
yield "Name", force_text(o.__class__.__name__)
is_module = inspect.ismodule(o)
yield "IsModule", is_module
if not is_module:
module = inspect.getmodule(o)
if module:
yield "Module", force_text(module.__name__)
yield "IsClass", inspect.isclass(o),
yield "IsFunction", inspect.isfunction(o),
yield "IsMethod", inspect.ismethod(o),
yield "IsCallable", callable(o),
if callable(o):
try:
yield "Arguments", map(force_text,
first(inspect.getargspec(o)))
except TypeError:
#this function can fail with TypeError unsupported callable
pass
def _get_traceback_frames(traceback, exc_value, context_lines=7):
def explicit_or_implicit_cause(exc_value):
explicit = getattr(exc_value, "__cause__", None)
implicit = getattr(exc_value, "__context__", None)
return explicit or implicit
# Get the exception and all its causes
exceptions = []
while exc_value:
exceptions.append(exc_value)
exc_value = explicit_or_implicit_cause(exc_value)
# No exceptions were supplied to ExceptionReporter
if exceptions:
# In case there's just one exception, take the traceback from self.tb
exc_value = exceptions.pop()
tb = traceback if not exceptions else exc_value.__traceback__
while tb is not None:
# Support for __traceback_hide__ which is used by a few libraries
# to hide internal frames.
if tb.tb_frame.f_locals.get("__traceback_hide__"):
tb = tb.tb_next
continue
filename = tb.tb_frame.f_code.co_filename
function = tb.tb_frame.f_code.co_name
lineno = tb.tb_lineno - 1
loader = tb.tb_frame.f_locals.get(
"__loader__") or tb.tb_frame.f_globals.get("__loader__")
module_name = tb.tb_frame.f_globals.get("__name__") or ""
pre_context_lineno, pre_context, context_line, post_context = _get_lines_from_file(
filename, lineno, context_lines, loader, module_name)
if pre_context_lineno is None:
pre_context_lineno = lineno
pre_context = []
context_line = "<source code not available>"
post_context = []
yield {
"filename": filename and force_text(filename) or None,
"function": function and force_text(function) or None,
"lineno": lineno + 1,
"variables": tb.tb_frame.f_locals,
"pre_context": pre_context,
"context_line": context_line,
"post_context": post_context,
"pre_context_lineno": pre_context_lineno + 1,
}
# If the traceback for current exception is consumed, try the
# other exception.
if not tb.tb_next and exceptions:
exc_value = exceptions.pop()
tb = exc_value.__traceback__
else:
tb = tb.tb_next
def report(self):
path = tempfile.gettempdir()
benchmarks = [(c, self.expression_handler(c)) for c in self.complexity]
self.table_line("dumping results in %s" % path)
self.table_line()
# running export to do all lazy loadings
binary_deserialize(export(1, target_format="wxf"))
self.table_line("* Binary deserialize")
self.table_line()
self.table_line(
"Memory",
*(force_text(c).ljust(self.col_size) for c in self.complexity))
self.table_divider(len(self.complexity) + 1)
for label, opts in (("wxf", dict()), ("wxf zip", dict(compress=True))):
self.table_line(
label,
*(self.formatted_time(
binary_deserialize,
export(expr, target_format="wxf", **opts))
for complexity, expr in benchmarks))
self.table_line()
self.table_line("* Export")
self.table_line()
for title, stream_generator in self.stream_generators(path):
self.table_line(
title,
*(force_text(c).ljust(self.col_size) for c in self.complexity))
self.table_divider(len(self.complexity) + 1)
for key in ("expr", "array"):
for label, export_format, opts in (
("wl", "wl", dict()),
("wxf", "wxf", dict()),
("wxf zip", "wxf", dict(compress=True)),
):
if key == "expr" or (key == "array" and not label == "wl"):
self.table_line(
key == "expr" and label or "%s %s" % (label, key),
*(self.formatted_time(export,
expr[key],
stream=stream_generator(
complexity,
export_format),
target_format=export_format,
**opts)
for complexity, expr in benchmarks))
self.table_line()
def test_wolfram_api_from_session(self):
api = (self.api_owner, "api/private/stringreverse")
apicall = self.cloud_session.wolfram_api_call(api)
apicall.set_parameter("str", "abcde")
future = apicall.perform_future()
res = future.result()
self.assertEqual('"edcba"', force_text(res.get()))
res = apicall.perform()
self.assertEqual('"edcba"', force_text(res.get()))
def report(self):
path = tempfile.gettempdir()
benchmarks = [(c, self.complexity_handler(c)) for c in self.complexity]
self.print("dumping results in", path)
# running export to do all lazy loadings
export(1)
for title, stream_generator in (
("Memory", lambda complexity: None),
(
"File",
lambda complexity: os.path.join(
path,
"benchmark-test-%s.%s" % (force_text(complexity).zfill(7), export_format),
),
),
):
self.table_line(
title, *(force_text(c).ljust(self.col_size) for c in self.complexity)
)
self.table_divider(len(self.complexity) + 1)
for label, export_format, opts in (
("wl", "wl", dict()),
("wxf", "wxf", dict()),
("wxf zip", "wxf", dict(compress=True)),
):
self.table_line(
label,
*(
self.formatted_time(
expr,
stream=stream_generator(complexity),
target_format=export_format,
**opts
)
for complexity, expr in benchmarks
)
)
self.table_line()
self.table_line()
def test_wolfram_api_call_async(self):
api = (self.api_owner, 'api/private/stringreverse')
apicall = WolframAPICall(self.cloud_session, api)
apicall.set_parameter('str', 'abcde')
future = apicall.perform_future()
res = future.result()
self.assertEqual('"edcba"', force_text(res.get()))
def serialize_bytes(self, bytes, as_byte_array=not six.PY2):
if as_byte_array:
return (WXF_CONSTANTS.BinaryString, varint_bytes(len(bytes)),
bytes)
else:
return self.serialize_string(
force_text(bytes, encoding="iso8859-1"))
def test_zero_dimension(self):
provider = WXFExprProvider(NumPyWXFEncoder())
arr = numpy.ndarray([2, 0, 3])
with self.assertRaises(Exception) as err:
next(provider.provide_wxfexpr(arr))
self.assertEqual(force_text(err.exception), "Dimensions must be positive integers.")
def table_line(self, *iterable):
self.print(
*(
force_text(c).ljust(i and self.col_size or self.title_size)
for i, c in enumerate(iterable)
)
)
def serialize_bytes(self, bytes, as_byte_array=not six.PY2):
if as_byte_array:
yield WXF_CONSTANTS.BinaryString
yield varint_bytes(len(bytes))
yield bytes
else:
for token in self.serialize_string(force_text(bytes, encoding="iso8859-1")):
yield token
def str_trim(o, max_char=80):
"""Return the string representation of an object, trimmed to keep up to `max_char` characters.
"""
as_str = force_text(o)
if len(as_str) > max_char:
return "%s...(%i more)" % (as_str[:max_char], len(as_str) - max_char)
else:
return as_str
def __init__(self, name):
if isinstance(name, six.binary_type):
self.name = force_text(name)
elif isinstance(name, six.text_type):
self.name = name
else:
raise ValueError(
"Symbol name should be %s not %s. You provided: %s" %
(six.text_type.__name__, name.__class__.__name__, name))
def aiohttp_request_meta(request, post):
yield "Method", request.method
yield "Scheme", request.url.scheme
yield "Domain", request.url.host
yield "Port", force_text(request.url.port)
yield "PathString", request.url.path
yield "QueryString", request.url.query_string
yield "Headers", tuple(wl.Rule(k, v) for k, v in request.headers.items())
yield "MultipartElements", tuple(
wl.Rule(k, to_multipart(v)) for k, v in post.items())
def serialize_bytes(self, bytes, as_byte_array=not six.PY2):
# by default we are serializing as_byte_array for PY3,
# py2 is by default using strings
if as_byte_array:
return self.serialize_function(
self.serialize_symbol(b"ByteArray"),
((b'"', base64.b64encode(bytes), b'"'), ))
else:
return self.serialize_string(force_text(bytes, "iso-8859-1"))
def _get_lines_from_file(filename,
lineno,
context_lines,
loader=None,
module_name=None):
"""
Return context_lines before and after lineno from file.
Return (pre_context_lineno, pre_context, context_line, post_context).
"""
source = None
if loader is not None and hasattr(loader, "get_source"):
try:
source = loader.get_source(module_name)
except ImportError:
pass
if source is not None:
source = source.splitlines()
if source is None:
try:
with open(filename, "rb") as fp:
source = fp.read().splitlines()
except (OSError, IOError):
pass
if source is None:
return None, [], None, []
# If we just read the source from a file, or if the loader did not
# apply tokenize.detect_encoding to decode the source into a
# string, then we should do that ourselves.
if isinstance(source[0], six.binary_type):
encoding = "ascii"
for line in source[:2]:
# File coding may be specified. Match pattern from PEP-263
# (http://www.python.org/dev/peps/pep-0263/)
match = re.search(br"coding[:=]\s*([-\w.]+)", line)
if match:
encoding = match.group(1).decode("ascii")
break
source = [force_text(sline, encoding, "replace") for sline in source]
lineno = min(lineno, len(source) - 1)
lower_bound = max(0, lineno - context_lines)
upper_bound = lineno + context_lines
pre_context = source[lower_bound:lineno]
context_line = source[lineno]
post_context = source[lineno + 1:upper_bound]
return lower_bound, pre_context, context_line, post_context
def write(self, message):
messages = force_text(message).split("\n")
if len(messages) == 1:
self.current_line.extend(messages)
else:
self.current_line.append(messages.pop(0))
rest = messages.pop(-1)
self.lines.extend(messages)
self.flush()
if rest:
self.current_line.append(rest)
def to_multipart(v, namegetter=identity, filegetter=identity):
if isinstance(v, six.string_types):
return {"ContentString": v, "InMemory": True}
destdir = os.path.join(tempfile.gettempdir(), force_text(uuid.uuid4()))
os.mkdir(destdir)
with open(os.path.join(destdir, namegetter(v)), "wb") as dest:
shutil.copyfileobj(filegetter(v), dest)
return {
"FileName": dest.name,
"InMemory": False,
"OriginalFileName": namegetter(v)
}
def start_zmq_instance(port=None, write_to_stdout=True, **opts):
# make a reply socket
sock = zmq.Context.instance().socket(zmq.PAIR)
# now bind to a open port on localhost
if port:
sock.bind("tcp://127.0.0.1:%s" % port)
else:
sock.bind_to_random_port("tcp://127.0.0.1")
if write_to_stdout:
sys.stdout.write(force_text(sock.getsockopt(zmq.LAST_ENDPOINT)))
sys.stdout.write(os.linesep) # writes \n
sys.stdout.flush()
return sock
def evaluate_message(input=None, return_type=None, args=None, **opts):
__traceback_hidden_variables__ = True
result = None
if isinstance(input, six.string_types):
result = execute_from_string(input, **opts)
if isinstance(args, (list, tuple)):
# then we have a function call to do
# first get the function object we need to call
result = result(*args)
if return_type == "string":
# bug 354267 repr returns a 'str' even on py2 (i.e. bytes).
result = force_text(repr(result))
return result
def test_force_encoding(self):
self.assertEqual(force_text("aà"), "aà")
self.assertEqual(force_text(abs), "<built-in function abs>")
self.assertEqual(force_text(b"a\xc3\xa0"), "aà")
self.assertEqual(force_text(memoryview(b"abc")), "abc")
self.assertEqual(force_text(bytearray(b"abc")), "abc")
self.assertEqual(force_bytes(b"abc"), b"abc")
self.assertEqual(force_bytes(abs), b"<built-in function abs>")
self.assertEqual(force_bytes("aà"), b"a\xc3\xa0")
self.assertEqual(force_text(force_bytes("aà")), "aà")
self.assertEqual(force_bytes(memoryview(b"abc")), b"abc")
self.assertEqual(force_bytes(bytearray(b"abc")), b"abc")
def test_section_api_call_no_param(self):
url = "api/private/requesterid"
response = self.cloud_session.call((self.api_owner, url))
self.assertIn(self.api_owner, force_text(response.get()))
def test_section_api_call_one_param(self):
url = "api/private/stringreverse"
response = self.cloud_session.call((self.api_owner, url),
input_parameters={"str": "abcde"})
self.assertEqual('"edcba"', force_text(response.get()))
def safe_wl_execute(function,
args=(),
opts={},
export_opts={},
exception_class=WolframLanguageException):
__traceback_hidden_variables__ = True
try:
return export(function(*args, **opts), **export_opts)
except Exception as export_exception:
try:
try:
# The user can provide an exception class, and it can be broken, in which case we are running another
# try / except to return errors that are happening during class serialization
if isinstance(export_exception, WolframLanguageException):
try:
export_exception.set_traceback(*sys.exc_info())
return export(export_exception, **export_opts)
except Exception:
pass
if exception_class is WolframLanguageException:
return export(
WolframLanguageException(export_exception,
exec_info=sys.exc_info()),
**export_opts)
# A custom error class might fail, if this is happening then we can try to use the built in one
return export(
exception_class(export_exception,
exec_info=sys.exc_info()), **export_opts)
except Exception as exception_export_err:
return export(
WolframLanguageException(exception_export_err,
exec_info=sys.exc_info()),
target_format=export_opts.get('target_format',
DEFAULT_FORMAT),
encoder=
'wolframclient.serializers.encoders.builtin.encoder',
)
except Exception as unknown_exception:
# This is the last resort.
# Everything went wrong, including the code that was supposed to return a traceback, or the custom
# normalizer is doing something it should not. This should never happen.
try:
return export(
wl.Failure(
"PythonFailure", {
"MessageTemplate":
safe_force_text(unknown_exception),
"MessageParameters": {},
"FailureCode":
safe_force_text(
unknown_exception.__class__.__name__),
"Traceback":
force_text(traceback.format_exc())
}),
target_format=export_opts.get('target_format',
DEFAULT_FORMAT),
encoder=
'wolframclient.serializers.encoders.builtin.encoder',
)
except Exception:
# Something went worst.
# this might happen with import errors / syntax errors in third party pluging that are loading the
# exporter and doing some real damage to the dispatcher we are using.
return DEFAULT_UNKNOWN_FAILURE[export_opts.get(
'target_format', DEFAULT_FORMAT)]
def test_wolfram_api_call_str(self):
api = (self.api_owner, "api/private/stringreverse")
apicall = WolframAPICall(self.cloud_session, api)
apicall.set_parameter("str", "abcde")
res = apicall.perform().get()
self.assertEqual('"edcba"', force_text(res))
def _paginate(i, line):
return "%s. %s" % (force_text(i).rjust(4), line)
def loads(s, **kwargs):
if isinstance(s, six.binary_type):
s = force_text(s)
return json_loads(s)
def stream_generators(self, path):
yield "Memory", lambda complexity, export_format, path=path: None
yield "File", lambda complexity, export_format, path=path: os.path.join(
path, "benchmark-test-%s.%s" %
(force_text(complexity).zfill(7), export_format))
def execute_from_file(path, *args, **opts):
with open(path, "r") as f:
return execute_from_string(force_text(f.read()), *args, **opts)
def table_line(self, *iterable):
self.print(*(force_text(c).ljust(self.col_size) for c in iterable))