def testUnicodeBytes(self):
foo = T.flatten('foo bar')
self.assertEquals(foo, T.flatten(u'foo bar'))
self.assertEquals(str(foo.tag), 's')
self.assertEquals(T.unflatten(foo.bytes, 'y'), b'foo bar')
self.assertEquals(T.unflatten(*T.flatten(b'foo bar', ['y'])),
b'foo bar')
def packetStream(packetHandler):
"""A generator that assembles packets.
This is the version 2 packet protocol for labrad, with request numbers.
We use the standard library module struct to decode the data. The
leading '>' in the format strings indicates big-endian data, 'I'
is equivalent to U32, and 'B' is equivalent to U8.
"""
buf = ''
while True:
# get packet header (20 bytes)
while len(buf) < 20:
buf += yield 0
hdr, buf = buf[:20], buf[20:]
context, request, source, length = T.unflatten(hdr, HEADER_TYPE)
# get packet data
while len(buf) < length:
buf += yield 0
s, buf = buf[:length], buf[length:]
# unflatten the data
records = []
s = T.Buffer(s)
while len(s):
ID, tag, data = T.unflatten(s, RECORD_TYPE)
rec = ID, T.unflatten(data, tag)
records.append(rec)
packetHandler(source, context, request, records)
def packetStream(packetHandler):
"""A generator that assembles packets.
This is the version 2 packet protocol for labrad, with request numbers.
We use the standard library module struct to decode the data. The
leading '>' in the format strings indicates big-endian data, 'I'
is equivalent to U32, and 'B' is equivalent to U8.
"""
buf = ''
while True:
# get packet header (20 bytes)
while len(buf) < 20:
buf += yield 0
hdr, buf = buf[:20], buf[20:]
context, request, source, length = T.unflatten(hdr, HEADER_TYPE)
# get packet data
while len(buf) < length:
buf += yield 0
s, buf = buf[:length], buf[length:]
# unflatten the data
records = []
s = T.Buffer(s)
while len(s):
ID, tag, data = T.unflatten(s, RECORD_TYPE)
rec = ID, T.unflatten(data, tag)
records.append(rec)
packetHandler(source, context, request, records)
def unflattenRecords(data, endianness='>'):
"""Unflatten a list of records from the data segment of a packet"""
records = []
s = T.Buffer(data)
while len(s):
ID, tag, data = T.unflatten(s, RECORD_TYPE, endianness)
rec = ID, T.unflatten(data, tag, endianness)
records.append(rec)
return records
def labrad_urldecode(data_url):
if data_url.startswith(DATA_URL_PREFIX):
# decode parameter data from dataurl
all_bytes = base64.urlsafe_b64decode(data_url[len(DATA_URL_PREFIX):])
t, data_bytes = T.unflatten(all_bytes, 'ss')
data = T.unflatten(data_bytes, t)
return data
else:
raise ValueError("Trying to labrad_urldecode data that doesn't start "
"with prefix: {}".format(DATA_URL_PREFIX))
def labrad_urldecode(data_url):
if data_url.startswith(DATA_URL_PREFIX):
# decode parameter data from dataurl
all_bytes = base64.urlsafe_b64decode(data_url[len(DATA_URL_PREFIX):])
t, data_bytes = T.unflatten(all_bytes, 'ss')
data = T.unflatten(data_bytes, t)
return data
else:
raise ValueError("Trying to labrad_urldecode data that doesn't start "
"with prefix: {}".format(DATA_URL_PREFIX))
def unflattenPacket(data, endianness='>'):
"""Unflatten a single labrad packet"""
hdr, rest = data[:20], data[20:]
context, request, source, length = T.unflatten(hdr, HEADER_TYPE, endianness)
assert len(rest) == length
records = unflattenRecords(rest, endianness)
return context, request, source, records
def testDefaultFlatAndBack(self):
"""
Test roundtrip python->LabRAD->python conversion.
No type requirements are given in these tests. In other words, we allow
pylabrad to choose a default type for flattening.
In this test, we expect A == unflatten(*flatten(A)). In other words,
we expect the default type chosen for each object to unflatten as
an object equal to the one originally flattened.
"""
tests = [
# simple types
None,
True, False,
1, -1, 2, -2, 0x7FFFFFFF, -0x80000000,
1L, 2L, 3L, 4L, 0L, 0xFFFFFFFFL,
'', 'a', '\x00\x01\x02\x03',
datetime.now(),
# values
5.0,
Value(6, ''),
Value(7, 'ms'),
8+0j,
Complex(9+0j, ''),
Complex(10+0j, 'GHz'),
# ValueArray and ndarray
# These types should be invariant under flattening followed by
# unflattening. Note, however, that since eg. [1, 2, 3] will
# unflatten as ndarray with dtype=int32, we do not put lists
# in this test.
U.ValueArray([1, 2, 3], 'm'),
U.ValueArray([1j, 2j, 3j], 's'),
np.array([1, 3, 4], dtype='int32'),
np.array([1.1, 2.2, 3.3]),
# clusters
(1, True, 'a'),
((1, 2), ('a', False)),
# lists
[],
#[1, 2, 3, 4],
#[1L, 2L, 3L, 4L],
[[]],
[['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']],
# more complex stuff
[(1L, 'a'), (2L, 'b')],
]
for data_in in tests:
data_out = T.unflatten(*T.flatten(data_in))
if isinstance(data_in, U.ValueArray):
self.assertTrue(data_in.allclose(data_out))
elif isinstance(data_in, np.ndarray):
np.testing.assert_array_equal(data_out, data_in)
else:
self.assertEqual(data_in, data_out)
def testDefaultFlatAndBackNonIdentical(self):
"""
Test flattening/unflattening of objects which change type.
No type requirements are given in these tests. In other words, we allow
pylabrad to choose a default type for flattening.
In this test, we do not expect A == unflatten(*flatten(A)). This is
mostly because list of numbers, both with an without units, should
unflatten to ndarray or ValueArray, rather than actual python lists.
"""
def compareValueArrays(a, b):
"""I check near equality of two ValueArrays"""
self.assertTrue(a.allclose(b))
tests = [
([1, 2, 3], np.array([1, 2, 3], dtype="int32"), np.testing.assert_array_equal),
([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype="float64"), np.testing.assert_array_almost_equal),
(np.array([3, 4], dtype="int32"), np.array([3, 4], dtype="int32"), np.testing.assert_array_equal),
(np.array([1.2, 3.4]), np.array([1.2, 3.4]), np.testing.assert_array_almost_equal),
([Value(1.0, "m"), Value(3.0, "m")], ValueArray([1.0, 3.0], "m"), compareValueArrays),
([Value(1.0, "m"), Value(10, "cm")], ValueArray([1.0, 0.1], "m"), compareValueArrays),
(ValueArray([1, 2], "Hz"), ValueArray([1, 2], "Hz"), compareValueArrays),
(ValueArray([1.0, 2], ""), np.array([1.0, 2]), np.testing.assert_array_almost_equal),
# Numpy scalar types
(np.bool8(True), True, self.assertEqual),
]
for input, expected, comparison_func in tests:
unflat = T.unflatten(*T.flatten(input))
if isinstance(unflat, np.ndarray):
self.assertEqual(unflat.dtype, expected.dtype)
comparison_func(unflat, expected)
def packetStream(packetHandler, endianness='>'):
"""A generator that assembles packets.
Accepts a function packetHandler that will be called with four arguments
whenever a packet is completed: source, context, request, records.
"""
buf = b''
while True:
# get packet header (20 bytes)
while len(buf) < 20:
buf += yield 0
hdr, buf = buf[:20], buf[20:]
context, request, source, length = T.unflatten(hdr,
HEADER_TYPE,
endianness=endianness)
# get packet data
while len(buf) < length:
buf += yield 0
s, buf = buf[:length], buf[length:]
# unflatten the data
records = unflattenRecords(s, endianness=endianness)
packetHandler(source, context, request, records)
def testNumpySupport(self):
"""Test flattening and unflattening of numpy arrays"""
import numpy as np
# TODO: flesh this out with more array types
a = np.array([1,2,3,4,5])
b = T.unflatten(*T.flatten(a)).asarray
self.assertTrue(np.all(a == b))
def performance_test():
for j in range(20):
unflattened = []
for s in str_be:
unflattened.append(types.unflatten(*s, endianness=">"))
str_le = []
for ufl in unflattened:
str_le.append(types.flatten(ufl, endianness="<"))
def testFlatAndBackWithTypeRequirements(self):
tests = [
([1, 2, 3], ["*i"], np.array([1, 2, 3]), np.testing.assert_array_equal),
([1, 2], ["*v[]"], np.array([1, 2]), np.testing.assert_array_almost_equal),
([1.1, 2.0], ["*v[]"], np.array([1.1, 2.0], dtype="float64"), np.testing.assert_array_almost_equal),
]
for input, types, expected, comparison_func in tests:
flat = T.flatten(input, types)
unflat = T.unflatten(*flat)
comparison_func(expected, unflat)
def getPar(i):
sec = 'Parameter %d' % (i+1)
label = S.get(sec, 'Label', raw=True)
raw = S.get(sec, 'Data', raw=True)
if raw.startswith(DATA_URL_PREFIX):
# decode parameter data from dataurl
all_bytes = base64.urlsafe_b64decode(raw[len(DATA_URL_PREFIX):])
t, data_bytes = T.unflatten(all_bytes, 'ss')
data = T.unflatten(data_bytes, t)
else:
# old parameters may have been saved using repr
try:
data = T.evalLRData(raw)
except RuntimeError:
if raw.endswith('None)'):
data = T.evalLRData(raw[0:-5] + '"")')
else:
raise
return dict(label=label, data=data)
def testNumpySupport(self):
"""Test flattening and unflattening of numpy arrays"""
# TODO: flesh this out with more array types
a = np.array([1, 2, 3, 4, 5], dtype='int32')
b = T.unflatten(*T.flatten(a))
self.assertTrue(np.all(a == b))
self.assertTrue(T.flatten(np.int32(5))[0] == b'\x00\x00\x00\x05')
self.assertTrue(T.flatten(np.int64(-5))[0] == b'\xff\xff\xff\xfb')
self.assertTrue(len(T.flatten(np.float64(3.15))[0]) == 8)
with self.assertRaises(T.FlatteningError):
T.flatten(np.int64(-5), T.TUInt())
def testNumpySupport(self):
"""Test flattening and unflattening of numpy arrays"""
# TODO: flesh this out with more array types
a = np.array([1, 2, 3, 4, 5], dtype='int32')
b = T.unflatten(*T.flatten(a))
self.assertTrue(np.all(a == b))
self.assertTrue(T.flatten(np.int32(5))[0] == b'\x00\x00\x00\x05')
self.assertTrue(T.flatten(np.int64(-5))[0] == b'\xff\xff\xff\xfb')
self.assertTrue(len(T.flatten(np.float64(3.15))[0]) == 8)
with self.assertRaises(T.FlatteningError):
T.flatten(np.int64(-5), T.TUInt())
def testFlatAndBackWithTypeRequirements(self):
tests = [([1, 2,
3], ['*i'], np.array([1, 2,
3]), np.testing.assert_array_equal),
([1, 2], ['*v[]'], np.array([1, 2]),
np.testing.assert_array_almost_equal),
([1.1, 2.], ['*v[]'], np.array([1.1, 2.], dtype='float64'),
np.testing.assert_array_almost_equal)]
for input, types, expected, comparison_func in tests:
flat = T.flatten(input, types)
unflat = T.unflatten(*flat)
comparison_func(expected, unflat)
def test_correctness(endianness=">"):
"""
Convert each data element to a string, then unflatten/flatten it and see if we
have the same string. This is easier than trying to compare arbitrary data structures
to see if they have been preserved.
"""
slist = []
for d in data:
ot = types.getType(d)
s, tt = types.flatten(d, ot, endianness)
slist.append((s, tt))
for idx, s in enumerate(slist):
new_data = types.unflatten(*s, endianness=endianness)
new_string = types.flatten(new_data, types.getType(new_data), endianness)
if new_string != s:
print "Data mismatch on index %d with byte order %s" % (idx, endianness)
def testFlatAndBack(self):
"""Test roundtrip python->LabRAD->python conversion."""
tests = [
# simple types
None,
True,
False,
1,
-1,
2,
-2,
1L,
2L,
3L,
4L,
'',
'a',
'\x00\x01\x02\x03',
datetime.now(),
# values
5.0,
T.Value(6, ''),
T.Value(7, 'ms'),
8 + 0j,
T.Complex(9 + 0j, ''),
T.Complex(10 + 0j, 'GHz'),
# clusters
(1, True, 'a'),
((1, 2), ('a', False)),
# lists
[],
[1, 2, 3, 4],
[1L, 2L, 3L, 4L],
[[]],
[['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']],
# more complex stuff
[(1L, 'a'), (2L, 'b')],
]
for data_in in tests:
#print data_in, T.flatten(data_in)
data_out = T.unflatten(*T.flatten(data_in))
self.assertEquals(data_in, data_out)
def testDefaultFlatAndBackNonIdentical(self):
"""
Test flattening/unflattening of objects which change type.
No type requirements are given in these tests. In other words, we allow
pylabrad to choose a default type for flattening.
In this test, we do not expect A == unflatten(*flatten(A)). This is
mostly because list of numbers, both with an without units, should
unflatten to ndarray or ValueArray, rather than actual python lists.
"""
def compareValueArrays(a, b):
"""I check near equality of two ValueArrays"""
self.assertTrue(a.allclose(b))
tests = [
([1, 2,
3], np.array([1, 2, 3],
dtype='int32'), np.testing.assert_array_equal),
([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype='float64'),
np.testing.assert_array_almost_equal),
(np.array([3, 4], dtype='int32'), np.array([3, 4], dtype='int32'),
np.testing.assert_array_equal),
(np.array([1.2, 3.4]), np.array([1.2, 3.4]),
np.testing.assert_array_almost_equal),
([Value(1.0, 'm'),
Value(3.0, 'm')], ValueArray([1.0, 3.0],
'm'), compareValueArrays),
([Value(1.0, 'm'),
Value(10, 'cm')], ValueArray([1.0, 0.1],
'm'), compareValueArrays),
(ValueArray([1, 2], 'Hz'), ValueArray([1, 2],
'Hz'), compareValueArrays),
(ValueArray([1.0, 2], ''), np.array([1.0, 2]),
np.testing.assert_array_almost_equal),
# Numpy scalar types
(np.bool8(True), True, self.assertEqual)
]
for input, expected, comparison_func in tests:
unflat = T.unflatten(*T.flatten(input))
if isinstance(unflat, np.ndarray):
self.assertEqual(unflat.dtype, expected.dtype)
comparison_func(unflat, expected)
def packetStream(packetHandler, endianness='>'):
"""A generator that assembles packets.
Accepts a function packetHandler that will be called with four arguments
whenever a packet is completed: source, context, request, records.
"""
buf = ''
while True:
# get packet header (20 bytes)
while len(buf) < 20:
buf += yield 0
hdr, buf = buf[:20], buf[20:]
context, request, source, length = T.unflatten(hdr, HEADER_TYPE, endianness=endianness)
# get packet data
while len(buf) < length:
buf += yield 0
s, buf = buf[:length], buf[length:]
# unflatten the data
records = unflattenRecords(s, endianness=endianness)
packetHandler(source, context, request, records)
def testFlatAndBack(self):
"""Test roundtrip python->LabRAD->python conversion."""
tests = [
# simple types
None,
True, False,
1, -1, 2, -2,
1L, 2L, 3L, 4L,
'', 'a', '\x00\x01\x02\x03',
datetime.now(),
# values
5.0,
T.Value(6, ''),
T.Value(7, 'ms'),
8+0j,
T.Complex(9+0j, ''),
T.Complex(10+0j, 'GHz'),
# clusters
(1, True, 'a'),
((1, 2), ('a', False)),
# lists
[],
[1, 2, 3, 4],
[1L, 2L, 3L, 4L],
[[]],
[['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']],
# more complex stuff
[(1L, 'a'), (2L, 'b')],
]
for data_in in tests:
#print data_in, T.flatten(data_in)
data_out = T.unflatten(*T.flatten(data_in))
self.assertEqual(data_in, data_out)
def testUnicodeBytes(self):
foo = T.flatten("foo bar")
self.assertEquals(foo, T.flatten(u"foo bar"))
self.assertEquals(str(foo.tag), "s")
self.assertEquals(T.unflatten(foo.bytes, "y"), "foo bar")
self.assertEquals(T.unflatten(*T.flatten("foo bar", ["y"])), "foo bar")
def testUnicodeBytes(self):
foo = T.flatten('foo bar')
self.assertEquals(foo, T.flatten(u'foo bar'))
self.assertEquals(str(foo.tag), 's')
self.assertEquals(T.unflatten(foo.bytes, 'y'), 'foo bar')
self.assertEquals(T.unflatten(*T.flatten('foo bar', ['y'])), 'foo bar')
def testBooleanArrayFlattening(self):
flat = T.flatten([True, False, True])
unflat = T.unflatten(*flat)
flat2 = T.flatten(unflat)
unflat2 = T.unflatten(*flat2)
np.testing.assert_array_equal(unflat, unflat2)
def testDefaultFlatAndBack(self):
"""
Test roundtrip python->LabRAD->python conversion.
No type requirements are given in these tests. In other words, we allow
pylabrad to choose a default type for flattening.
In this test, we expect A == unflatten(*flatten(A)). In other words,
we expect the default type chosen for each object to unflatten as
an object equal to the one originally flattened.
"""
tests = [
# simple types
None,
True,
False,
1,
-1,
2,
-2,
0x7FFFFFFF,
-0x80000000,
1L,
2L,
3L,
4L,
0L,
0xFFFFFFFFL,
'',
'a',
'\x00\x01\x02\x03',
datetime.now(),
# values
5.0,
Value(6, ''),
Value(7, 'ms'),
8 + 0j,
Complex(9 + 0j, ''),
Complex(10 + 0j, 'GHz'),
# ValueArray and ndarray
# These types should be invariant under flattening followed by
# unflattening. Note, however, that since eg. [1, 2, 3] will
# unflatten as ndarray with dtype=int32, we do not put lists
# in this test.
U.ValueArray([1, 2, 3], 'm'),
U.ValueArray([1j, 2j, 3j], 's'),
np.array([1, 3, 4], dtype='int32'),
np.array([1.1, 2.2, 3.3]),
# clusters
(1, True, 'a'),
((1, 2), ('a', False)),
# lists
[],
#[1, 2, 3, 4],
#[1L, 2L, 3L, 4L],
[[]],
[['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']],
# more complex stuff
[(1L, 'a'), (2L, 'b')],
]
for data_in in tests:
data_out = T.unflatten(*T.flatten(data_in))
if isinstance(data_in, U.ValueArray):
self.assertTrue(data_in.allclose(data_out))
elif isinstance(data_in, np.ndarray):
np.testing.assert_array_equal(data_out, data_in)
else:
self.assertEqual(data_in, data_out)
def testBooleanArrayFlattening(self):
flat = T.flatten([True, False, True])
unflat = T.unflatten(*flat)
flat2 = T.flatten(unflat)
unflat2 = T.unflatten(*flat2)
np.testing.assert_array_equal(unflat, unflat2)
def testFlattenIntArrayToValueArray(self):
x = np.array([1, 2, 3, 4], dtype='int64')
flat = T.flatten(x, '*v')
y = T.unflatten(*flat)
self.assertTrue(np.all(x == y))
def testFlattenIntArrayToValueArray(self):
x = np.array([1, 2, 3, 4], dtype='int64')
flat = T.flatten(x, '*v')
y = T.unflatten(*flat)
self.assertTrue(np.all(x == y))
