def test_sync_from_storage(self): io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair() data_set_consumer = DataSet(storage_reader=io_reader, name='consumer') some_array = DataArray('some_array', 'label', shape=(5, 1)) io_writer.sync_add_data_array_to_storage(some_array) data_set_consumer.sync_from_storage(-1) self.assertTrue(hasattr(data_set_consumer, 'some_array'))
def test_sync_from_storage_meta_data(self): mock_queue = MagicMock() with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch( 'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \ patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue): reader = MongoDataSetIOReader(name='test') thread.assert_called_once() mock_io.assert_called_once_with('test', None, create_if_not_found=False, collection='data_sets', database='qilib') data_set = DataSet(storage_reader=reader) mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': { 'metadata': { 'name': 'test_name' } } } } data_set.sync_from_storage(-1) self.assertEqual('test_name', data_set.name) mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': { 'metadata.default_array_name': 'some_array' } } } data_set.sync_from_storage(-1) self.assertEqual('some_array', data_set.default_array_name)
def test_integrate_with_data_set_io_add_array(self): io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair() data_set_consumer = DataSet(storage_reader=io_reader) some_array = DataArray('some_array', 'label', shape=(5, 5)) data_set_producer = DataSet(storage_writer=io_writer) data_set_producer.add_array(some_array) data_set_consumer.sync_from_storage(-1) self.assertTrue(hasattr(data_set_consumer, 'some_array')) self.assertEqual('some_array', data_set_consumer.some_array.name) self.assertEqual((5, 5), data_set_consumer.some_array.shape) self.assertIsNot(some_array, data_set_consumer.some_array)
def test_integrate_with_data_set_io_add_data(self): io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair() data_set_consumer = DataSet(storage_reader=io_reader) some_array = DataArray('some_array', 'label', shape=(5, 5)) data_set_producer = DataSet(storage_writer=io_writer, data_arrays=some_array) data_set_producer.add_data((0, 0), {'some_array': 42}) data_set_producer.add_data((1, 1), {'some_array': 25}) data_set_consumer.sync_from_storage(-1) self.assertTrue(hasattr(data_set_consumer, 'some_array')) self.assertIsNot(some_array, data_set_consumer.some_array) self.assertEqual(42, data_set_consumer.some_array[0][0]) self.assertEqual(25, data_set_consumer.some_array[1][1])
def test_sync_from_storage_array_update(self): mock_queue = MagicMock() with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch( 'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \ patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue): reader = MongoDataSetIOReader(name='test') thread.assert_called_once() mock_io.assert_called_once_with('test', None, create_if_not_found=False, collection='data_sets', database='qilib') data_set = DataSet(storage_reader=reader) data_array = DataArray(name='test_array', label='lab', shape=(2, 2)) data_set.add_array(data_array) mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': { 'array_updates': [[[0, 0], { 'test_array': 42 }], [[0, 1], { 'test_array': 25 }]] } } } data_set.sync_from_storage(-1) self.assertListEqual([42, 25], list(data_set.test_array[0])) mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': { 'array_updates.1': [1, { 'test_array': [67, 67] }] } } } data_set.sync_from_storage(-1) self.assertListEqual([67, 67], list(data_set.test_array[1]))
def test_metadata_triggers_update(self): name = 'Bobo' user_data = PythonJsonStructure(data='plata', snapshot=False) timestamp = datetime.datetime(2019, 12, 24) default_array_name = 'ThatsAGoodName' io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair() data_set_consumer = DataSet(storage_reader=io_reader) data_set_producer = DataSet(storage_writer=io_writer) data_set_producer.name = name data_set_producer.user_data = user_data data_set_producer.time_stamp = timestamp data_set_producer.default_array_name = default_array_name data_set_consumer.sync_from_storage(-1) self.assertEqual(name, data_set_consumer.name) self.assertDictEqual(user_data, data_set_consumer.user_data) self.assertEqual(timestamp, data_set_consumer.time_stamp) self.assertEqual(default_array_name, data_set_consumer.default_array_name)
def test_sync_from_storage_array(self): mock_queue = MagicMock() with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch( 'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \ patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue): mock_io.decode_numpy_array = MongoDataSetIO.decode_numpy_array reader = MongoDataSetIOReader(name='test') thread.assert_called_once() mock_io.assert_called_once_with('test', None, create_if_not_found=False, collection='data_sets', database='qilib') data_set = DataSet(storage_reader=reader) set_array = DataArray(name='setter', label='for_testing', is_setpoint=True, preset_data=np.array(range(0, 2))) set_array[0] = 42 set_array[1] = 25 data_array = DataArray(name='test_array', label='lab', unit='V', is_setpoint=False, set_arrays=[set_array], shape=(2, )) update_data = { "data_arrays": { "setter": { "name": set_array.name, "label": set_array.label, "unit": set_array.unit, "is_setpoint": set_array.is_setpoint, "set_arrays": [array.name for array in set_array.set_arrays], "preset_data": MongoDataSetIO.encode_numpy_array(set_array) } } } mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': update_data } } data_set.sync_from_storage(-1) update_data = { "data_arrays.test_array": { "name": data_array.name, "label": data_array.label, "unit": data_array.unit, "is_setpoint": data_array.is_setpoint, "set_arrays": [array.name for array in data_array.set_arrays], "preset_data": MongoDataSetIO.encode_numpy_array(data_array) } } mock_queue.get.return_value = { 'updateDescription': { 'updatedFields': update_data } } data_set.sync_from_storage(-1) self.assertEqual('test_array', data_set.test_array.name) self.assertEqual('lab', data_set.test_array.label) self.assertEqual('V', data_set.test_array.unit) self.assertFalse(data_set.test_array.is_setpoint) self.assertEqual('setter', data_array.set_arrays[0].name) self.assertListEqual([42, 25], list(data_array.set_arrays[0])) data_array[0] = 255 update_data["data_arrays.test_array"][ "preset_data"] = MongoDataSetIO.encode_numpy_array(data_array) data_set.sync_from_storage(-1) self.assertEqual(255, data_set.test_array[0])