def export_compare(self, o, wl=None, wxf=None, **kwargs):
if wl:
res = export(o, target_format="wl", **kwargs)
self.assertEqual(res, wl)
if wxf:
res = export(o, target_format="wxf", **kwargs)
self.assertEqual(res, wxf)
def test_serialization_custom(self):
class MyStuff(object):
def __init__(self, *stuff):
self.stuff = stuff
def normalizer(o):
if isinstance(o, six.integer_types):
return 'o'
if isinstance(o, MyStuff):
return wl.RandomThings(*o.stuff)
return o
expr = [1, 2, 'a', {1: "a"}, MyStuff(1, 2, MyStuff(1, 'a'))]
normalized = [
"o", "o", "a", {
"o": "a"
},
wl.RandomThings("o", "o", wl.RandomThings("o", "a"))
]
for export_format in available_formats:
with self.assertRaises(NotImplementedError) as context:
export(expr, normalizer=identity, target_format=export_format)
self.assertEqual(
export(expr,
normalizer=normalizer,
target_format=export_format),
export(normalized,
normalizer=identity,
target_format=export_format),
)
def report(self):
path = tempfile.gettempdir()
benchmarks = [(c, self.expression_handler(c)) for c in self.complexity]
self.table_line("dumping results in %s" % path)
self.table_line()
# running export to do all lazy loadings
binary_deserialize(export(1, target_format="wxf"))
self.table_line("* Binary deserialize")
self.table_line()
self.table_line(
"Memory",
*(force_text(c).ljust(self.col_size) for c in self.complexity))
self.table_divider(len(self.complexity) + 1)
for label, opts in (("wxf", dict()), ("wxf zip", dict(compress=True))):
self.table_line(
label,
*(self.formatted_time(
binary_deserialize,
export(expr, target_format="wxf", **opts))
for complexity, expr in benchmarks))
self.table_line()
self.table_line("* Export")
self.table_line()
for title, stream_generator in self.stream_generators(path):
self.table_line(
title,
*(force_text(c).ljust(self.col_size) for c in self.complexity))
self.table_divider(len(self.complexity) + 1)
for key in ("expr", "array"):
for label, export_format, opts in (
("wl", "wl", dict()),
("wxf", "wxf", dict()),
("wxf zip", "wxf", dict(compress=True)),
):
if key == "expr" or (key == "array" and not label == "wl"):
self.table_line(
key == "expr" and label or "%s %s" % (label, key),
*(self.formatted_time(export,
expr[key],
stream=stream_generator(
complexity,
export_format),
target_format=export_format,
**opts)
for complexity, expr in benchmarks))
self.table_line()
def pth2wxf(path):
pth = torch.load(path, map_location=torch.device('cpu'))
npy = {
key: value.numpy()
for key, value in pth.items() if not re.match('.*_tracked$', key)
}
wxf.export(npy, path + '.wxf', target_format='wxf')
def edge_model(dataset):
data = torch.load('checkpoints/' + dataset + '/EdgeModel_gen.pth',
map_location='cpu')
npy = {key: value.numpy() for key, value in data['generator'].items()}
wxf.export(npy, 'EdgeModel_' + dataset + '.wxf', target_format='wxf')
generator = EdgeGenerator()
generator.load_state_dict(data['generator'])
torch.save(generator, 'EdgeModel_' + dataset + '.pth')
def caffe2wxf(path):
model = opencv.readNetFromCaffe(path + ".prototxt", path + ".caffemodel")
layers = model.getLayerNames()
npy = {}
for i in layers:
try:
npy[i] = model.getParam(i)
except Exception:
pass
wxf.export(npy, path + '.wxf', target_format='wxf')
def pkl2wxf(path):
file = open(path, 'rb')
objs = []
while True:
try:
objs.append(pkl.load(file))
except EOFError:
break
file.close()
print(objs)
wxf.export(objs, path + '.wxf', target_format='wxf')
def report(self):
path = tempfile.gettempdir()
benchmarks = [(c, self.complexity_handler(c)) for c in self.complexity]
self.print("dumping results in", path)
# running export to do all lazy loadings
export(1)
for title, stream_generator in (
("Memory", lambda complexity: None),
(
"File",
lambda complexity: os.path.join(
path,
"benchmark-test-%s.%s" % (force_text(complexity).zfill(7), export_format),
),
),
):
self.table_line(
title, *(force_text(c).ljust(self.col_size) for c in self.complexity)
)
self.table_divider(len(self.complexity) + 1)
for label, export_format, opts in (
("wl", "wl", dict()),
("wxf", "wxf", dict()),
("wxf zip", "wxf", dict(compress=True)),
):
self.table_line(
label,
*(
self.formatted_time(
expr,
stream=stream_generator(complexity),
target_format=export_format,
**opts
)
for complexity, expr in benchmarks
)
)
self.table_line()
self.table_line()
def _encode_inputs_as_wxf(inputs, multipart, **kwargs):
for name, value in inputs.items():
yield "%s__wxf" % name, _to_multipart(name,
export(value,
target_format="wxf",
**kwargs),
multipart=multipart)
def test_register_twice_force(self):
@wolfram_encoder.dispatch(subsubfoo, replace_existing=True)
def encode_subsubfoo_again(s, o):
return s.serialize_symbol("subsubfooFORCE")
wl = export(subsubfoo())
self.assertEqual(wl, b"subsubfooFORCE")
def _call_evaluation_api(self, expr, **kwargs):
data = export(expr, target_format="wl", **kwargs)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"Sending expression to cloud server for evaluation: %s", data)
response = self._post(self.evaluation_api_url, body=data)
return WolframCloudEvaluationWXFResponse(response)
def _encode_inputs_as_wl(form_data, inputs, **kwargs):
for name, value in inputs.items():
# avoid double encoding of strings '\"string\"'.
if isinstance(value, six.string_types):
form_data.add_field(name, value)
else:
form_data.add_field(name, export(value, target_format="wl", **kwargs))
async def _call_evaluation_api(self, expr, **kwargs):
data = aiohttp.BytesPayload(export(expr, target_format="wl", **kwargs))
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"Sending expression to cloud server for evaluation: %s", data)
response = await self._post(self.evaluation_api_url, data=data)
return WolframEvaluationWXFResponseAsync(response)
def do_evaluate_future(self, expr, result_update_callback=None, **kwargs):
future = futures.Future()
wxf = export(self.normalize_input(expr), target_format="wxf", **kwargs)
self.kernel_controller.evaluate_future(
wxf, future, result_update_callback=result_update_callback, **kwargs
)
return future
def test_png_mode_I(self):
with PIL.open(path_to_file_in_data_dir('5x2.png')) as image:
self.assertEqual(
export(image, target_format='wl'),
b'ImportByteArray[ByteArray["iVBORw0KGgoAAAANSUhEUgAAAAUAAAACEAAAAADlkZXCAAAAH0lEQVR4nGP0+P39rf6+ky9/R7Aoen2+9shDWSRCHwCO7ws73c3PRQAAAABJRU5ErkJggg=="], "PNG"]'
)
def test_bad_incomplete_consumer(self):
with self.assertRaises(WolframParserException) as e:
binary_deserialize(export([1, 2, 3], target_format="wxf"),
consumer=self.BadIncompleteConsumer())
self.assertEqual(
e.msg,
"Input data does not represent a valid expression in WXF format. Some expressions are imcomplete.",
)
def test_bad_greedy_consumer(self):
with self.assertRaises(WolframParserException) as e:
binary_deserialize(export([1, 2, 3], target_format="wxf"),
consumer=self.BadGreedyConsumer())
self.assertEqual(
e.msg,
"Input data does not represent a valid expression in WXF format. Expecting more input data.",
)
def test_export(self):
for value in (
1,
2,
"aaaa",
2.0,
{
1: 2
},
[1, 2, 3],
["hello", decimal.Decimal("1.23")],
wl.Foo,
wl.Foo(2, wl.Context.Internal),
):
self.serialize_compare(value, export(value, target_format="wxf"))
self.assertEqual(
export(Association(enumerate("abc")), target_format="wxf"),
export(
wl.Association(*(wl.Rule(i, v) for i, v in enumerate("abc"))),
target_format="wxf",
),
)
self.assertEqual(
export(wlexpr("2+2"), target_format="wxf"),
export(wl.ToExpression("2+2"), target_format="wxf"),
)
self.assertEqual(
export(wl.Foo(wlexpr("2+2"), 1, 2), target_format="wxf"),
export(wl.Foo(wl.ToExpression("2+2"), 1, 2), target_format="wxf"),
)
def caffe2wxf(path):
data = dict()
model = caffe.NetParameter()
model.ParseFromString(open(path, 'rb').read())
get_array = lambda p: npy.array(p.data, dtype='float32').reshape(p.shape.
dim)
def add_array(node):
for i in range(len(node.blobs)):
data[node.name + "_" + str(i + 1)] = get_array(node.blobs[i])
def add_node(layers):
for i in range(len(layers)):
add_array(layers[i])
add_node(model.layer)
wxf.export(data, path + '.wxf', target_format='wxf')
def handle(self, format, **opts):
data = self.create_data(**opts)
if format:
self.print(export(dict(data), target_format=format))
else:
for k, v in data:
if isinstance(v, float):
v = "%.4f" % v
self.table_line(k, v)
def test_mode_L(self):
a = numpy.arange(10).reshape((2, 5))
img = PIL.fromarray(a, mode='L')
out = export(img, target_format='wl')
self.assertTrue(
out ==
b'Image[BinaryDeserialize[ByteArray["ODrCEAICBQAAAAAAAAAAAQA="]], "Byte", Rule[ColorSpace, "Grayscale"], Rule[Interleaving, True]]'
or out ==
b'Image[BinaryDeserialize[ByteArray["ODrCEAICBQAAAAAAAAAAAQA="]], "Byte", Rule[Interleaving, True], Rule[ColorSpace, "Grayscale"]]'
)
def compare_serializer(self,
serializer,
array,
value,
test_export_wxf=False):
serializer.serialize(array)
self.assertEqual(serializer._writer.getvalue(), value)
if test_export_wxf:
self.assertEqual(export(array, target_format="wxf"), value)
def _encode_inputs_as_wl(inputs, multipart, **kwargs):
for name, value in inputs.items():
# avoid double encoding of strings '\"string\"'.
if isinstance(value, six.string_types):
yield name, _to_multipart(name, value, multipart=multipart)
else:
yield name, _to_multipart(
name,
export(value, target_format='wl', **kwargs),
multipart=multipart)
def test_generators(self):
array = NumericArray((i for i in range(10)), "Integer8", shape=(10, ))
self.assertEqual(
export(numpy.arange(10).astype(numpy.int8), target_format="wxf"),
export(array, target_format="wxf"),
)
self.assertEqual(len(array), 10)
array = NumericArray((i for i in range(10)), "Integer8", shape=(5, 2))
self.assertEqual(
export(numpy.arange(10).astype(numpy.int8).reshape(5, 2), target_format="wxf"),
export(array, target_format="wxf"),
)
self.assertEqual(len(array), 10)
def test_bool_img(self):
a = numpy.array([[1, 0], [0, 1]], dtype='bool')
img = PIL.fromarray(a)
out = export(img, target_format='wl')
self.assertTrue(
out ==
b'Image[BinaryDeserialize[ByteArray["ODrCEAICAgEAAAA="]], "Bit", Rule[ColorSpace, Automatic], Rule[Interleaving, True]]'
or out ==
b'Image[BinaryDeserialize[ByteArray["ODrCEAICAgEAAAA="]], "Bit", Rule[Interleaving, True], Rule[ColorSpace, Automatic]]'
)
def process_generate_httpresponse_expression(response):
if isinstance(response, dict):
if not response.get("BodyByteArray", None):
# empty byte array is returning a an empty list, we need an empty byte array
response["BodyByteArray"] = b""
return response
return {
"BodyByteArray": export(response, target_format="wl"),
"Headers": (wl.Rule("content-type", "text/plain;charset=utf-8"), ),
"StatusCode": 500,
}
def test_python_array(self):
for array, numpy_type, wl_type in (
([True, False, True, False, True, False], numpy.int8, "Integer8"),
([1, 2, 3, 4, 5, 6], numpy.int8, "Integer8"),
([1, 2, 3, 4, 5, 6], numpy.int32, "Integer32"),
([1, 2, 3, 4, 5, 6], numpy.int64, "Integer64"),
([1.2, 2.3, 3, 4, 5, 6], numpy.float32, "Real32"),
([1.2, 2.3, 3, 4, 5, 6], numpy.float64, "Real64"),
):
for shape in ((3, 2), None):
arr = numpy.array(array, numpy_type)
if shape:
arr = arr.reshape(shape)
self.assertEqual(
export(arr, target_format="wxf"),
export(NumericArray(array, wl_type, shape=shape), target_format="wxf"),
)
def test_numpy_integers(self):
int8 = [numpy.int8(127), numpy.int8(-128)]
self.assertEqual(export(int8), b"{127, -128}")
uint8 = [numpy.uint8(0), numpy.uint8(255)]
self.assertEqual(export(uint8), b"{0, 255}")
int16 = [numpy.int16(32767), numpy.int16(-32768)]
self.assertEqual(export(int16), b"{32767, -32768}")
ints = [
numpy.uint16(65535),
numpy.int32(-2147483648),
numpy.uint32(4294967296),
numpy.int64(-9223372036854775808),
numpy.uint64(18446744073709551615),
]
self.assertEqual(
export(ints),
b"{65535, -2147483648, 0, -9223372036854775808, 18446744073709551615}",
)
def test_export_with_encoder(self):
#very similar code is used by safe_wl_execute, we need to make sure that we can pass the builtin encoder and a very simple
#data dump and the code keeps working
self.assertEqual(
export(
wl.Failure("PythonFailure",
{'MessageTemplate': ['baz', ('bar', 'bad')]}),
target_format='wl',
encoder='wolframclient.serializers.encoders.builtin.encoder'),
b'Failure["PythonFailure", <|"MessageTemplate" -> {"baz", {"bar", "bad"}}|>]'
)
def test_export(self):
# checking that export is able to return bytes if no second argument is provided
self.assertEqual(export(2), b"2")
self.assertEqual(export("foo"), b'"foo"')
fd, path = tempfile.mkstemp()
# close the file descriptor but keep the path. Prevent error on Windows.
os.close(fd)
for test in ["foo", wl.Symbol, {"a": [1, 2, 3], 2: 2}]:
for export_format in available_formats:
expected = export(test, target_format=export_format)
# checking that export is able to write to a path if a string is provided
export(test, path, target_format=export_format)
with open(path, "rb") as stream:
self.assertEqual(stream.read(), expected)
# checking that export is writing to a byteio
stream = six.BytesIO()
export(test, stream, target_format=export_format)
stream.seek(0)
self.assertEqual(stream.read(), expected)
# checking that export is able to write to a filelike object
with open(path, "wb") as stream:
export(test, stream, target_format=export_format)
with open(path, "rb") as stream:
self.assertEqual(stream.read(), expected)
os.remove(path)
