def testUnicodeBytes(self): foo = T.flatten('foo bar') self.assertEquals(foo, T.flatten(u'foo bar')) self.assertEquals(str(foo.tag), 's') self.assertEquals(T.unflatten(foo.bytes, 'y'), b'foo bar') self.assertEquals(T.unflatten(*T.flatten(b'foo bar', ['y'])), b'foo bar')
def packetStream(packetHandler): """A generator that assembles packets. This is the version 2 packet protocol for labrad, with request numbers. We use the standard library module struct to decode the data. The leading '>' in the format strings indicates big-endian data, 'I' is equivalent to U32, and 'B' is equivalent to U8. """ buf = '' while True: # get packet header (20 bytes) while len(buf) < 20: buf += yield 0 hdr, buf = buf[:20], buf[20:] context, request, source, length = T.unflatten(hdr, HEADER_TYPE) # get packet data while len(buf) < length: buf += yield 0 s, buf = buf[:length], buf[length:] # unflatten the data records = [] s = T.Buffer(s) while len(s): ID, tag, data = T.unflatten(s, RECORD_TYPE) rec = ID, T.unflatten(data, tag) records.append(rec) packetHandler(source, context, request, records)
def unflattenRecords(data, endianness='>'): """Unflatten a list of records from the data segment of a packet""" records = [] s = T.Buffer(data) while len(s): ID, tag, data = T.unflatten(s, RECORD_TYPE, endianness) rec = ID, T.unflatten(data, tag, endianness) records.append(rec) return records
def labrad_urldecode(data_url): if data_url.startswith(DATA_URL_PREFIX): # decode parameter data from dataurl all_bytes = base64.urlsafe_b64decode(data_url[len(DATA_URL_PREFIX):]) t, data_bytes = T.unflatten(all_bytes, 'ss') data = T.unflatten(data_bytes, t) return data else: raise ValueError("Trying to labrad_urldecode data that doesn't start " "with prefix: {}".format(DATA_URL_PREFIX))
def unflattenPacket(data, endianness='>'): """Unflatten a single labrad packet""" hdr, rest = data[:20], data[20:] context, request, source, length = T.unflatten(hdr, HEADER_TYPE, endianness) assert len(rest) == length records = unflattenRecords(rest, endianness) return context, request, source, records
def testDefaultFlatAndBack(self): """ Test roundtrip python->LabRAD->python conversion. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we expect A == unflatten(*flatten(A)). In other words, we expect the default type chosen for each object to unflatten as an object equal to the one originally flattened. """ tests = [ # simple types None, True, False, 1, -1, 2, -2, 0x7FFFFFFF, -0x80000000, 1L, 2L, 3L, 4L, 0L, 0xFFFFFFFFL, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, Value(6, ''), Value(7, 'ms'), 8+0j, Complex(9+0j, ''), Complex(10+0j, 'GHz'), # ValueArray and ndarray # These types should be invariant under flattening followed by # unflattening. Note, however, that since eg. [1, 2, 3] will # unflatten as ndarray with dtype=int32, we do not put lists # in this test. U.ValueArray([1, 2, 3], 'm'), U.ValueArray([1j, 2j, 3j], 's'), np.array([1, 3, 4], dtype='int32'), np.array([1.1, 2.2, 3.3]), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], #[1, 2, 3, 4], #[1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: data_out = T.unflatten(*T.flatten(data_in)) if isinstance(data_in, U.ValueArray): self.assertTrue(data_in.allclose(data_out)) elif isinstance(data_in, np.ndarray): np.testing.assert_array_equal(data_out, data_in) else: self.assertEqual(data_in, data_out)
def testDefaultFlatAndBackNonIdentical(self): """ Test flattening/unflattening of objects which change type. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we do not expect A == unflatten(*flatten(A)). This is mostly because list of numbers, both with an without units, should unflatten to ndarray or ValueArray, rather than actual python lists. """ def compareValueArrays(a, b): """I check near equality of two ValueArrays""" self.assertTrue(a.allclose(b)) tests = [ ([1, 2, 3], np.array([1, 2, 3], dtype="int32"), np.testing.assert_array_equal), ([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype="float64"), np.testing.assert_array_almost_equal), (np.array([3, 4], dtype="int32"), np.array([3, 4], dtype="int32"), np.testing.assert_array_equal), (np.array([1.2, 3.4]), np.array([1.2, 3.4]), np.testing.assert_array_almost_equal), ([Value(1.0, "m"), Value(3.0, "m")], ValueArray([1.0, 3.0], "m"), compareValueArrays), ([Value(1.0, "m"), Value(10, "cm")], ValueArray([1.0, 0.1], "m"), compareValueArrays), (ValueArray([1, 2], "Hz"), ValueArray([1, 2], "Hz"), compareValueArrays), (ValueArray([1.0, 2], ""), np.array([1.0, 2]), np.testing.assert_array_almost_equal), # Numpy scalar types (np.bool8(True), True, self.assertEqual), ] for input, expected, comparison_func in tests: unflat = T.unflatten(*T.flatten(input)) if isinstance(unflat, np.ndarray): self.assertEqual(unflat.dtype, expected.dtype) comparison_func(unflat, expected)
def packetStream(packetHandler, endianness='>'): """A generator that assembles packets. Accepts a function packetHandler that will be called with four arguments whenever a packet is completed: source, context, request, records. """ buf = b'' while True: # get packet header (20 bytes) while len(buf) < 20: buf += yield 0 hdr, buf = buf[:20], buf[20:] context, request, source, length = T.unflatten(hdr, HEADER_TYPE, endianness=endianness) # get packet data while len(buf) < length: buf += yield 0 s, buf = buf[:length], buf[length:] # unflatten the data records = unflattenRecords(s, endianness=endianness) packetHandler(source, context, request, records)
def testNumpySupport(self): """Test flattening and unflattening of numpy arrays""" import numpy as np # TODO: flesh this out with more array types a = np.array([1,2,3,4,5]) b = T.unflatten(*T.flatten(a)).asarray self.assertTrue(np.all(a == b))
def performance_test(): for j in range(20): unflattened = [] for s in str_be: unflattened.append(types.unflatten(*s, endianness=">")) str_le = [] for ufl in unflattened: str_le.append(types.flatten(ufl, endianness="<"))
def testFlatAndBackWithTypeRequirements(self): tests = [ ([1, 2, 3], ["*i"], np.array([1, 2, 3]), np.testing.assert_array_equal), ([1, 2], ["*v[]"], np.array([1, 2]), np.testing.assert_array_almost_equal), ([1.1, 2.0], ["*v[]"], np.array([1.1, 2.0], dtype="float64"), np.testing.assert_array_almost_equal), ] for input, types, expected, comparison_func in tests: flat = T.flatten(input, types) unflat = T.unflatten(*flat) comparison_func(expected, unflat)
def getPar(i): sec = 'Parameter %d' % (i+1) label = S.get(sec, 'Label', raw=True) raw = S.get(sec, 'Data', raw=True) if raw.startswith(DATA_URL_PREFIX): # decode parameter data from dataurl all_bytes = base64.urlsafe_b64decode(raw[len(DATA_URL_PREFIX):]) t, data_bytes = T.unflatten(all_bytes, 'ss') data = T.unflatten(data_bytes, t) else: # old parameters may have been saved using repr try: data = T.evalLRData(raw) except RuntimeError: if raw.endswith('None)'): data = T.evalLRData(raw[0:-5] + '"")') else: raise return dict(label=label, data=data)
def testNumpySupport(self): """Test flattening and unflattening of numpy arrays""" # TODO: flesh this out with more array types a = np.array([1, 2, 3, 4, 5], dtype='int32') b = T.unflatten(*T.flatten(a)) self.assertTrue(np.all(a == b)) self.assertTrue(T.flatten(np.int32(5))[0] == b'\x00\x00\x00\x05') self.assertTrue(T.flatten(np.int64(-5))[0] == b'\xff\xff\xff\xfb') self.assertTrue(len(T.flatten(np.float64(3.15))[0]) == 8) with self.assertRaises(T.FlatteningError): T.flatten(np.int64(-5), T.TUInt())
def testFlatAndBackWithTypeRequirements(self): tests = [([1, 2, 3], ['*i'], np.array([1, 2, 3]), np.testing.assert_array_equal), ([1, 2], ['*v[]'], np.array([1, 2]), np.testing.assert_array_almost_equal), ([1.1, 2.], ['*v[]'], np.array([1.1, 2.], dtype='float64'), np.testing.assert_array_almost_equal)] for input, types, expected, comparison_func in tests: flat = T.flatten(input, types) unflat = T.unflatten(*flat) comparison_func(expected, unflat)
def test_correctness(endianness=">"): """ Convert each data element to a string, then unflatten/flatten it and see if we have the same string. This is easier than trying to compare arbitrary data structures to see if they have been preserved. """ slist = [] for d in data: ot = types.getType(d) s, tt = types.flatten(d, ot, endianness) slist.append((s, tt)) for idx, s in enumerate(slist): new_data = types.unflatten(*s, endianness=endianness) new_string = types.flatten(new_data, types.getType(new_data), endianness) if new_string != s: print "Data mismatch on index %d with byte order %s" % (idx, endianness)
def testFlatAndBack(self): """Test roundtrip python->LabRAD->python conversion.""" tests = [ # simple types None, True, False, 1, -1, 2, -2, 1L, 2L, 3L, 4L, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, T.Value(6, ''), T.Value(7, 'ms'), 8 + 0j, T.Complex(9 + 0j, ''), T.Complex(10 + 0j, 'GHz'), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], [1, 2, 3, 4], [1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: #print data_in, T.flatten(data_in) data_out = T.unflatten(*T.flatten(data_in)) self.assertEquals(data_in, data_out)
def testDefaultFlatAndBackNonIdentical(self): """ Test flattening/unflattening of objects which change type. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we do not expect A == unflatten(*flatten(A)). This is mostly because list of numbers, both with an without units, should unflatten to ndarray or ValueArray, rather than actual python lists. """ def compareValueArrays(a, b): """I check near equality of two ValueArrays""" self.assertTrue(a.allclose(b)) tests = [ ([1, 2, 3], np.array([1, 2, 3], dtype='int32'), np.testing.assert_array_equal), ([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype='float64'), np.testing.assert_array_almost_equal), (np.array([3, 4], dtype='int32'), np.array([3, 4], dtype='int32'), np.testing.assert_array_equal), (np.array([1.2, 3.4]), np.array([1.2, 3.4]), np.testing.assert_array_almost_equal), ([Value(1.0, 'm'), Value(3.0, 'm')], ValueArray([1.0, 3.0], 'm'), compareValueArrays), ([Value(1.0, 'm'), Value(10, 'cm')], ValueArray([1.0, 0.1], 'm'), compareValueArrays), (ValueArray([1, 2], 'Hz'), ValueArray([1, 2], 'Hz'), compareValueArrays), (ValueArray([1.0, 2], ''), np.array([1.0, 2]), np.testing.assert_array_almost_equal), # Numpy scalar types (np.bool8(True), True, self.assertEqual) ] for input, expected, comparison_func in tests: unflat = T.unflatten(*T.flatten(input)) if isinstance(unflat, np.ndarray): self.assertEqual(unflat.dtype, expected.dtype) comparison_func(unflat, expected)
def packetStream(packetHandler, endianness='>'): """A generator that assembles packets. Accepts a function packetHandler that will be called with four arguments whenever a packet is completed: source, context, request, records. """ buf = '' while True: # get packet header (20 bytes) while len(buf) < 20: buf += yield 0 hdr, buf = buf[:20], buf[20:] context, request, source, length = T.unflatten(hdr, HEADER_TYPE, endianness=endianness) # get packet data while len(buf) < length: buf += yield 0 s, buf = buf[:length], buf[length:] # unflatten the data records = unflattenRecords(s, endianness=endianness) packetHandler(source, context, request, records)
def testFlatAndBack(self): """Test roundtrip python->LabRAD->python conversion.""" tests = [ # simple types None, True, False, 1, -1, 2, -2, 1L, 2L, 3L, 4L, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, T.Value(6, ''), T.Value(7, 'ms'), 8+0j, T.Complex(9+0j, ''), T.Complex(10+0j, 'GHz'), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], [1, 2, 3, 4], [1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: #print data_in, T.flatten(data_in) data_out = T.unflatten(*T.flatten(data_in)) self.assertEqual(data_in, data_out)
def testUnicodeBytes(self): foo = T.flatten("foo bar") self.assertEquals(foo, T.flatten(u"foo bar")) self.assertEquals(str(foo.tag), "s") self.assertEquals(T.unflatten(foo.bytes, "y"), "foo bar") self.assertEquals(T.unflatten(*T.flatten("foo bar", ["y"])), "foo bar")
def testUnicodeBytes(self): foo = T.flatten('foo bar') self.assertEquals(foo, T.flatten(u'foo bar')) self.assertEquals(str(foo.tag), 's') self.assertEquals(T.unflatten(foo.bytes, 'y'), 'foo bar') self.assertEquals(T.unflatten(*T.flatten('foo bar', ['y'])), 'foo bar')
def testBooleanArrayFlattening(self): flat = T.flatten([True, False, True]) unflat = T.unflatten(*flat) flat2 = T.flatten(unflat) unflat2 = T.unflatten(*flat2) np.testing.assert_array_equal(unflat, unflat2)
def testDefaultFlatAndBack(self): """ Test roundtrip python->LabRAD->python conversion. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we expect A == unflatten(*flatten(A)). In other words, we expect the default type chosen for each object to unflatten as an object equal to the one originally flattened. """ tests = [ # simple types None, True, False, 1, -1, 2, -2, 0x7FFFFFFF, -0x80000000, 1L, 2L, 3L, 4L, 0L, 0xFFFFFFFFL, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, Value(6, ''), Value(7, 'ms'), 8 + 0j, Complex(9 + 0j, ''), Complex(10 + 0j, 'GHz'), # ValueArray and ndarray # These types should be invariant under flattening followed by # unflattening. Note, however, that since eg. [1, 2, 3] will # unflatten as ndarray with dtype=int32, we do not put lists # in this test. U.ValueArray([1, 2, 3], 'm'), U.ValueArray([1j, 2j, 3j], 's'), np.array([1, 3, 4], dtype='int32'), np.array([1.1, 2.2, 3.3]), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], #[1, 2, 3, 4], #[1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: data_out = T.unflatten(*T.flatten(data_in)) if isinstance(data_in, U.ValueArray): self.assertTrue(data_in.allclose(data_out)) elif isinstance(data_in, np.ndarray): np.testing.assert_array_equal(data_out, data_in) else: self.assertEqual(data_in, data_out)
def testFlattenIntArrayToValueArray(self): x = np.array([1, 2, 3, 4], dtype='int64') flat = T.flatten(x, '*v') y = T.unflatten(*flat) self.assertTrue(np.all(x == y))