def testUnitTypes(self): """Test flattening with units. The flattening code should not do unit conversion, but should leave that up to the LabRAD manager to handle. Basically, for purposes of flattening, a unit is a unit. """ tests = [ (Value(5.0, 'ft'), ['v[m]'], 'v[ft]'), # real value array (U.ValueArray([1, 2, 3], ''), [], '*v[]'), (U.ValueArray([1, 2, 3], 'm'), ['*v[m]'], '*v[m]'), # complex value array (U.ValueArray([1j, 2j, 3j], ''), [], '*c[]'), (U.ValueArray([1j, 2j, 3j], 'm'), [], '*c[m]') ] for data, hints, tag in tests: self.assertEqual(T.flatten(data, hints)[1], T.parseTypeTag(tag)) # we disallow flattening a float to a value with units, # as this is a major source of bugs try: T.flatten(5.0, 'v[m]') except Exception: pass else: raise Exception('Cannot flatten float to value with units')
def testUnicodeBytes(self): foo = T.flatten('foo bar') self.assertEquals(foo, T.flatten(u'foo bar')) self.assertEquals(str(foo.tag), 's') self.assertEquals(T.unflatten(foo.bytes, 'y'), b'foo bar') self.assertEquals(T.unflatten(*T.flatten(b'foo bar', ['y'])), b'foo bar')
def testIntegerRanges(self): """Test flattening of out-of-range integer values""" tests = [(0x80000000, 'i'), (-0x80000001, 'i'), (0x100000000, 'w'), (-1, 'w')] for n, t in tests: with self.assertRaises(T.FlatteningError): T.flatten(n, t)
def getSettingInfoByName(self, serverID, settingName): """Get information about a setting using its name.""" packet = [(C.HELP, T.flatten((serverID, settingName), 'ws')), (C.LOOKUP, T.flatten((serverID, settingName), 'ws'))] resp = yield self._send(packet) description, accepts, returns, notes = resp[0][1] ID = resp[1][1][1] returnValue((description, accepts, returns, notes, ID))
def testFlattenArrayToClusterList(self): """Fail if trying to flatten a numpy array to type with incorrect shape. See https://github.com/labrad/pylabrad/issues/290. """ arr = np.arange(5, dtype='float64') with self.assertRaises(T.FlatteningError): T.flatten(arr, types=['*(v, v)'])
def getServerInfo(self, serverID): """Get information about a server.""" packet = [(C.HELP, T.flatten(serverID, 'w')), (C.SETTINGS_LIST, T.flatten(serverID, 'w'))] resp = yield self._send(packet) descr, notes = resp[0][1] settings = self._reorderIDList(resp[1][1]) returnValue((descr, notes, settings))
def testCanFlattenFlatData(self): x = ('this is a test', -42, [False, True]) flat = T.flatten(x) self.assertEquals(T.parseTypeTag(flat.tag), T.parseTypeTag('si*b')) flat2 = T.flatten(x) self.assertEquals(flat2, flat) flat3 = T.flatten(x, 'si*b') self.assertEquals(flat3, flat) with self.assertRaises(T.FlatteningError): T.flatten(x, 'sv')
def testIntegerRanges(self): """Test flattening of out-of-range integer values""" tests = [ (0x80000000, 'i'), (-0x80000001, 'i'), (0x100000000, 'w'), (-1, 'w') ] for n, t in tests: with self.assertRaises(T.FlatteningError): T.flatten(n, t)
def labrad_urlencode(data): if hasattr(T, 'FlatData'): # pylabrad 0.95+ flat_data = T.flatten(data) flat_cluster = T.flatten((str(flat_data.tag), flat_data.bytes), 'sy') all_bytes = flat_cluster.bytes else: data_bytes, t = T.flatten(data) all_bytes, _ = T.flatten((str(t), data_bytes), 'ss') data_url = DATA_URL_PREFIX + base64.urlsafe_b64encode(all_bytes) return data_url
def test_parameters(dv): """Create a dataset with parameters""" dv.new('test', ['x', 'y'], ['z']) for i in xrange(100): t = hydrant.randType(noneOkay=False) a = hydrant.randValue(t) name = 'param{}'.format(i) dv.add_parameter(name, a) b = dv.get_parameter(name) flat_a = T.flatten(a) flat_b = T.flatten(b) assert flat_a == flat_b
def hoseDown(setting, n=1000, silent=True): for _ in range(n): t = randType() v = randValue(t) if not silent: print t try: resp = setting(v) assert v == resp except: print 'problem:', str(t), repr(t) print str(T.flatten(v)[1]), str(T.flatten(resp)[1]) raise
def hoseDataVault(dv, n=1000, silent=True): for i in range(n): t = randType(noneOkay=False) v = randValue(t) if not silent: print t try: pname = 'p%03s' % i dv.add_parameter(pname, v) resp = dv.get_parameter(pname) assert v == resp except: print 'problem:', str(t), repr(t) print str(T.flatten(v)[1]), str(T.flatten(resp)[1]) raise
def getServerInfoWithSettings(self, serverID): """Get information about a server, including all of its settings.""" packet = [(C.HELP, T.flatten(serverID, 'w')), (C.SETTINGS_LIST, T.flatten(serverID, 'w'))] resp = yield self._send(packet) descr, notes = resp[0][1] settings = resp[1][1] packet = [(C.HELP, T.flatten((serverID, ID), 'ww')) for ID, name in settings] resp = yield self._send(packet) settingList = [] for s, r in zip(settings, resp): ID, name = s descr, accepts, returns, notes = r[1] settingList.append((name, ID, (descr, accepts, returns, notes))) returnValue((descr, notes, settingList))
def testDefaultFlatAndBackNonIdentical(self): """ Test flattening/unflattening of objects which change type. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we do not expect A == unflatten(*flatten(A)). This is mostly because list of numbers, both with an without units, should unflatten to ndarray or ValueArray, rather than actual python lists. """ def compareValueArrays(a, b): """I check near equality of two ValueArrays""" self.assertTrue(a.allclose(b)) tests = [ ([1, 2, 3], np.array([1, 2, 3], dtype="int32"), np.testing.assert_array_equal), ([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype="float64"), np.testing.assert_array_almost_equal), (np.array([3, 4], dtype="int32"), np.array([3, 4], dtype="int32"), np.testing.assert_array_equal), (np.array([1.2, 3.4]), np.array([1.2, 3.4]), np.testing.assert_array_almost_equal), ([Value(1.0, "m"), Value(3.0, "m")], ValueArray([1.0, 3.0], "m"), compareValueArrays), ([Value(1.0, "m"), Value(10, "cm")], ValueArray([1.0, 0.1], "m"), compareValueArrays), (ValueArray([1, 2], "Hz"), ValueArray([1, 2], "Hz"), compareValueArrays), (ValueArray([1.0, 2], ""), np.array([1.0, 2]), np.testing.assert_array_almost_equal), # Numpy scalar types (np.bool8(True), True, self.assertEqual), ] for input, expected, comparison_func in tests: unflat = T.unflatten(*T.flatten(input)) if isinstance(unflat, np.ndarray): self.assertEqual(unflat.dtype, expected.dtype) comparison_func(unflat, expected)
def testDefaultFlatAndBack(self): """ Test roundtrip python->LabRAD->python conversion. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we expect A == unflatten(*flatten(A)). In other words, we expect the default type chosen for each object to unflatten as an object equal to the one originally flattened. """ tests = [ # simple types None, True, False, 1, -1, 2, -2, 0x7FFFFFFF, -0x80000000, 1L, 2L, 3L, 4L, 0L, 0xFFFFFFFFL, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, Value(6, ''), Value(7, 'ms'), 8+0j, Complex(9+0j, ''), Complex(10+0j, 'GHz'), # ValueArray and ndarray # These types should be invariant under flattening followed by # unflattening. Note, however, that since eg. [1, 2, 3] will # unflatten as ndarray with dtype=int32, we do not put lists # in this test. U.ValueArray([1, 2, 3], 'm'), U.ValueArray([1j, 2j, 3j], 's'), np.array([1, 3, 4], dtype='int32'), np.array([1.1, 2.2, 3.3]), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], #[1, 2, 3, 4], #[1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: data_out = T.unflatten(*T.flatten(data_in)) if isinstance(data_in, U.ValueArray): self.assertTrue(data_in.allclose(data_out)) elif isinstance(data_in, np.ndarray): np.testing.assert_array_equal(data_out, data_in) else: self.assertEqual(data_in, data_out)
def testTypeHints(self): """Test conversion to specified allowed types.""" passingTests = [ # convert to default type (1, [], 'i'), # convert to first compatible type (1, ['s', 'w'], 'w'), (1, ['s', 'v'], 'v'), (1, ['s', 'v[m]'], 'v[m]'), # empty list gets type from hint ([], ['s', '*(ww)'], '*(ww)'), # handle unknown pieces inside clusters and lists (['a', 'b'], ['*?'], '*s'), ((1, 2, 'a'), ['ww?'], 'wws'), ((1, 1L), ['??'], 'iw'), ] failingTests = [ # no compatible types (5.0, ['s', 'b', 't', 'w', 'i'], 'v'), ] for data, hints, tag in passingTests: self.assertEquals(T.flatten(data, hints)[1], T.parseTypeTag(tag)) for data, hints, tag in failingTests: self.assertRaises(T.FlatteningError, T.flatten, data, hints)
def testTypeHints(self): """Test conversion to specified allowed types.""" passingTests = [ # convert to default type (1, [], 'i'), # convert to first compatible type (1, ['s', 'w'], 'w'), (1, ['s', 'v'], 'v'), (1*U.m, ['s', 'v[m]'], 'v[m]'), # empty list gets type from hint ([], ['s', '*(ww)'], '*(ww)'), # handle unknown pieces inside clusters and lists (['a', 'b'], ['*?'], '*s'), ((1, 2, 'a'), ['ww?'], 'wws'), ((1, 1L), ['??'], 'iw'), ] failingTests = [ # no compatible types (5.0, ['s', 'b', 't', 'w', 'i'], 'v'), ] for data, hints, tag in passingTests: self.assertEqual(T.flatten(data, hints)[1], T.parseTypeTag(tag)) for data, hints, tag in failingTests: self.assertRaises(T.FlatteningError, T.flatten, data, hints)
def extract(d, prefix=''): parameters = [] for key, val in sorted(d.items()): if ignore and key.startswith(ignore): continue elif val is None: continue elif isinstance(val, dict): parameters.extend(extract(val, prefix + key + '.')) else: try: types.flatten(val) parameters.append((prefix + key, val)) except Exception: pass # skip anything that cannot be flattened by LabRAD return parameters
def testTypeSpecialization(self): """Test specialization of the type during flattening.""" tests = [ # specialization without hints ([([], ), ([5.0], ), ([T.Value(5, 'm')], )], '*(*v[m])'), ] for data, tag in tests: self.assertEquals(T.flatten(data)[1], T.parseTypeTag(tag))
def testTypeSpecialization(self): """Test specialization of the type during flattening.""" tests = [ # specialization without hints ([([],), ([5.0],), ([T.Value(5, 'm')],)], '*(*v[m])'), ] for data, tag in tests: self.assertEquals(T.flatten(data)[1], T.parseTypeTag(tag))
def test_correctness(endianness=">"): """ Convert each data element to a string, then unflatten/flatten it and see if we have the same string. This is easier than trying to compare arbitrary data structures to see if they have been preserved. """ slist = [] for d in data: ot = types.getType(d) s, tt = types.flatten(d, ot, endianness) slist.append((s, tt)) for idx, s in enumerate(slist): new_data = types.unflatten(*s, endianness=endianness) new_string = types.flatten(new_data, types.getType(new_data), endianness) if new_string != s: print "Data mismatch on index %d with byte order %s" % (idx, endianness)
def testNumpySupport(self): """Test flattening and unflattening of numpy arrays""" import numpy as np # TODO: flesh this out with more array types a = np.array([1,2,3,4,5]) b = T.unflatten(*T.flatten(a)).asarray self.assertTrue(np.all(a == b))
def save(self): S = util.DVSafeConfigParser() sec = 'General' S.add_section(sec) S.set(sec, 'Created', time_to_str(self.created)) S.set(sec, 'Accessed', time_to_str(self.accessed)) S.set(sec, 'Modified', time_to_str(self.modified)) S.set(sec, 'Title', self.title) S.set(sec, 'Independent', repr(len(self.independents))) S.set(sec, 'Dependent', repr(len(self.dependents))) S.set(sec, 'Parameters', repr(len(self.parameters))) S.set(sec, 'Comments', repr(len(self.comments))) for i, ind in enumerate(self.independents): sec = 'Independent %d' % (i+1) S.add_section(sec) S.set(sec, 'Label', ind['label']) S.set(sec, 'Units', ind['units']) for i, dep in enumerate(self.dependents): sec = 'Dependent %d' % (i+1) S.add_section(sec) S.set(sec, 'Label', dep['label']) S.set(sec, 'Units', dep['units']) S.set(sec, 'Category', dep['category']) for i, par in enumerate(self.parameters): sec = 'Parameter %d' % (i+1) S.add_section(sec) S.set(sec, 'Label', par['label']) # encode the parameter value as a data-url data_bytes, t = T.flatten(par['data']) all_bytes, _ = T.flatten((str(t), data_bytes), 'ss') data_url = DATA_URL_PREFIX + base64.urlsafe_b64encode(all_bytes) S.set(sec, 'Data', data_url) sec = 'Comments' S.add_section(sec) for i, (time, user, comment) in enumerate(self.comments): time = time_to_str(time) S.set(sec, 'c%d' % i, repr((time, user, comment))) with open(self.infofile, 'w') as f: S.write(f)
def _doLogin(self, *ident): self.ident = ident # Store name, which is always the first identification param. self.name = ident[0] # Send identification. data = (1, ) + ident tag = 'w' + 's' * len(ident) flat = T.flatten(data, tag) self.ID = yield self._sendManagerRequest(0, flat)
def performance_test(): for j in range(20): unflattened = [] for s in str_be: unflattened.append(types.unflatten(*s, endianness=">")) str_le = [] for ufl in unflattened: str_le.append(types.flatten(ufl, endianness="<"))
def _doLogin(self, *ident): self.ident = ident # Store name, which is always the first identification param. self.name = ident[0] # Send identification. data = (1,) + ident tag = 'w' + 's'*len(ident) flat = T.flatten(data, tag) self.ID = yield self._sendManagerRequest(0, flat)
def flattenRecord(ID, data, types=[], endianness='>'): """Flatten a piece of data into a record with datatype and property.""" try: flat = T.flatten(data, types, endianness) except T.FlatteningError as e: e.msg = e.msg + "\nSetting ID %s." % (ID, ) raise flat_record = RECORD_TYPE.flatten((ID, str(flat.tag), bytes(flat.bytes)), endianness) return flat_record.bytes
def __call__(self, *args, **kw): """Send a request to this setting.""" tag = extractKey(kw, 'tag', None) or self.accepts if len(args) == 0: args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) r = yield self._server._send([(self.ID, flat)], **kw) returnValue(r[0][1])
def __setitem__(self, key, value): """Update existing parts of the packet, indexed by key. Note that if multiple records share the same key, they will all be updated. """ for i, rec in enumerate(self._packet): if key == rec.key: flat = T.flatten(value, rec.tag) self._packet[i] = rec._replace(data=value, flat=flat)
def testFlatAndBackWithTypeRequirements(self): tests = [ ([1, 2, 3], ["*i"], np.array([1, 2, 3]), np.testing.assert_array_equal), ([1, 2], ["*v[]"], np.array([1, 2]), np.testing.assert_array_almost_equal), ([1.1, 2.0], ["*v[]"], np.array([1.1, 2.0], dtype="float64"), np.testing.assert_array_almost_equal), ] for input, types, expected, comparison_func in tests: flat = T.flatten(input, types) unflat = T.unflatten(*flat) comparison_func(expected, unflat)
def flattenRecord(ID, data, types=[], endianness='>'): """Flatten a piece of data into a record with datatype and property.""" try: flat = T.flatten(data, types, endianness) except T.FlatteningError as e: e.msg = e.msg + "\nSetting ID %s." % (ID,) raise flat_record = RECORD_TYPE.flatten((ID, str(flat.tag), str(flat.bytes)), endianness) return flat_record.bytes
def wrapped(self, *args, **kw): key = extractKey(kw, 'key', None) tag = extractKey(kw, 'tag', None) or setting.accepts if len(args) == 0: args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) rec = PacketRecord(ID=setting.ID, data=args, tag=tag, flat=flat, key=key) self._packet.append(rec) return self
def wrapped(*args, **kw): key = kw.pop('key', None) tag = kw.pop('tag', None) or s.accepts if not len(args): args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) rec = PacketRecord(ID=s.ID, data=args, tag=tag, flat=flat, key=key) self._packet.append(rec) return self
def __call__(self, *args, **kw): """Send a request to this setting.""" tag = extractKey(kw, 'tag', None) or self.accepts if len(args) == 0: args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) d = self._server._send([(self.ID, flat)], **kw) d.addCallback(lambda r: r[0][1]) return d
def flattenRecord(ID, data, types=[], endianness='>'): """Flatten a piece of data into a record with datatype and property.""" if isinstance(data, T.FlatData): s, t = data else: try: s, t = T.flatten(data, types, endianness) except T.FlatteningError as e: e.msg = e.msg + "\nSetting ID %s." % (ID,) raise return RECORD_TYPE.__flatten__((ID, str(t), str(s)), endianness)[0]
def future(self, *args, **kw): wrap = kw.pop('wrap', True) tag = kw.pop('tag', None) or self.accepts if not len(args): args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) f = self._server._send([(self.ID, flat)], **kw) if wrap: f = map_future(f, lambda resp: resp[0][1]) return MutableFuture(f)
def future(self, *args, **kw): wrap = kw.pop('wrap', True) tag = kw.pop('tag', None) or self.accepts_type if not len(args): args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) f = self._server._send([(self.ID, flat)], **kw) if wrap: f = map_future(f, lambda resp: resp[0][1]) return f
def testFlatAndBackWithTypeRequirements(self): tests = [([1, 2, 3], ['*i'], np.array([1, 2, 3]), np.testing.assert_array_equal), ([1, 2], ['*v[]'], np.array([1, 2]), np.testing.assert_array_almost_equal), ([1.1, 2.], ['*v[]'], np.array([1.1, 2.], dtype='float64'), np.testing.assert_array_almost_equal)] for input, types, expected, comparison_func in tests: flat = T.flatten(input, types) unflat = T.unflatten(*flat) comparison_func(expected, unflat)
def __call__(self, *args, **kw): wait = kw.pop('wait', True) wrap = kw.pop('wrap', True) tag = kw.pop('tag', None) or self.accepts if not len(args): args = None elif len(args) == 1: args = args[0] flat = T.flatten(args, tag) future = self._server._send([(self.ID, flat)], **kw) if wrap: future.addCallback(lambda resp: resp[0][1]) return future.wait() if wait else future
def testUnitTypes(self): """Test flattening with units. The flattening code should not do unit conversion, but should leave that up to the LabRAD manager to handle. Basically, for purposes of flattening, a unit is a unit. """ tests = [ (5.0, ['v[m]'], 'v[m]'), (T.Value(5.0, 'ft'), ['v[m]'], 'v[ft]'), ] for data, hints, tag in tests: self.assertEquals(T.flatten(data, hints)[1], T.parseTypeTag(tag))
def testNumpySupport(self): """Test flattening and unflattening of numpy arrays""" # TODO: flesh this out with more array types a = np.array([1, 2, 3, 4, 5], dtype='int32') b = T.unflatten(*T.flatten(a)) self.assertTrue(np.all(a == b)) self.assertTrue(T.flatten(np.int32(5))[0] == b'\x00\x00\x00\x05') self.assertTrue(T.flatten(np.int64(-5))[0] == b'\xff\xff\xff\xfb') self.assertTrue(len(T.flatten(np.float64(3.15))[0]) == 8) with self.assertRaises(T.FlatteningError): T.flatten(np.int64(-5), T.TUInt())
def testFlatAndBack(self): """Test roundtrip python->LabRAD->python conversion.""" tests = [ # simple types None, True, False, 1, -1, 2, -2, 1L, 2L, 3L, 4L, '', 'a', '\x00\x01\x02\x03', datetime.now(), # values 5.0, T.Value(6, ''), T.Value(7, 'ms'), 8 + 0j, T.Complex(9 + 0j, ''), T.Complex(10 + 0j, 'GHz'), # clusters (1, True, 'a'), ((1, 2), ('a', False)), # lists [], [1, 2, 3, 4], [1L, 2L, 3L, 4L], [[]], [['a', 'bb', 'ccc'], ['dddd', 'eeeee', 'ffffff']], # more complex stuff [(1L, 'a'), (2L, 'b')], ] for data_in in tests: #print data_in, T.flatten(data_in) data_out = T.unflatten(*T.flatten(data_in)) self.assertEquals(data_in, data_out)
def testDefaultFlatAndBackNonIdentical(self): """ Test flattening/unflattening of objects which change type. No type requirements are given in these tests. In other words, we allow pylabrad to choose a default type for flattening. In this test, we do not expect A == unflatten(*flatten(A)). This is mostly because list of numbers, both with an without units, should unflatten to ndarray or ValueArray, rather than actual python lists. """ def compareValueArrays(a, b): """I check near equality of two ValueArrays""" self.assertTrue(a.allclose(b)) tests = [ ([1, 2, 3], np.array([1, 2, 3], dtype='int32'), np.testing.assert_array_equal), ([1.1, 2.2, 3.3], np.array([1.1, 2.2, 3.3], dtype='float64'), np.testing.assert_array_almost_equal), (np.array([3, 4], dtype='int32'), np.array([3, 4], dtype='int32'), np.testing.assert_array_equal), (np.array([1.2, 3.4]), np.array([1.2, 3.4]), np.testing.assert_array_almost_equal), ([Value(1.0, 'm'), Value(3.0, 'm')], ValueArray([1.0, 3.0], 'm'), compareValueArrays), ([Value(1.0, 'm'), Value(10, 'cm')], ValueArray([1.0, 0.1], 'm'), compareValueArrays), (ValueArray([1, 2], 'Hz'), ValueArray([1, 2], 'Hz'), compareValueArrays), (ValueArray([1.0, 2], ''), np.array([1.0, 2]), np.testing.assert_array_almost_equal), # Numpy scalar types (np.bool8(True), True, self.assertEqual) ] for input, expected, comparison_func in tests: unflat = T.unflatten(*T.flatten(input)) if isinstance(unflat, np.ndarray): self.assertEqual(unflat.dtype, expected.dtype) comparison_func(unflat, expected)