def test_sync_from_storage(self):
io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair()
data_set_consumer = DataSet(storage_reader=io_reader, name='consumer')
some_array = DataArray('some_array', 'label', shape=(5, 1))
io_writer.sync_add_data_array_to_storage(some_array)
data_set_consumer.sync_from_storage(-1)
self.assertTrue(hasattr(data_set_consumer, 'some_array'))
def test_sync_from_storage_meta_data(self):
mock_queue = MagicMock()
with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch(
'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \
patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue):
reader = MongoDataSetIOReader(name='test')
thread.assert_called_once()
mock_io.assert_called_once_with('test',
None,
create_if_not_found=False,
collection='data_sets',
database='qilib')
data_set = DataSet(storage_reader=reader)
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': {
'metadata': {
'name': 'test_name'
}
}
}
}
data_set.sync_from_storage(-1)
self.assertEqual('test_name', data_set.name)
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': {
'metadata.default_array_name': 'some_array'
}
}
}
data_set.sync_from_storage(-1)
self.assertEqual('some_array', data_set.default_array_name)
def test_integrate_with_data_set_io_add_array(self):
io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair()
data_set_consumer = DataSet(storage_reader=io_reader)
some_array = DataArray('some_array', 'label', shape=(5, 5))
data_set_producer = DataSet(storage_writer=io_writer)
data_set_producer.add_array(some_array)
data_set_consumer.sync_from_storage(-1)
self.assertTrue(hasattr(data_set_consumer, 'some_array'))
self.assertEqual('some_array', data_set_consumer.some_array.name)
self.assertEqual((5, 5), data_set_consumer.some_array.shape)
self.assertIsNot(some_array, data_set_consumer.some_array)
def test_integrate_with_data_set_io_add_data(self):
io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair()
data_set_consumer = DataSet(storage_reader=io_reader)
some_array = DataArray('some_array', 'label', shape=(5, 5))
data_set_producer = DataSet(storage_writer=io_writer,
data_arrays=some_array)
data_set_producer.add_data((0, 0), {'some_array': 42})
data_set_producer.add_data((1, 1), {'some_array': 25})
data_set_consumer.sync_from_storage(-1)
self.assertTrue(hasattr(data_set_consumer, 'some_array'))
self.assertIsNot(some_array, data_set_consumer.some_array)
self.assertEqual(42, data_set_consumer.some_array[0][0])
self.assertEqual(25, data_set_consumer.some_array[1][1])
def test_sync_from_storage_array_update(self):
mock_queue = MagicMock()
with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch(
'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \
patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue):
reader = MongoDataSetIOReader(name='test')
thread.assert_called_once()
mock_io.assert_called_once_with('test',
None,
create_if_not_found=False,
collection='data_sets',
database='qilib')
data_set = DataSet(storage_reader=reader)
data_array = DataArray(name='test_array',
label='lab',
shape=(2, 2))
data_set.add_array(data_array)
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': {
'array_updates': [[[0, 0], {
'test_array': 42
}], [[0, 1], {
'test_array': 25
}]]
}
}
}
data_set.sync_from_storage(-1)
self.assertListEqual([42, 25], list(data_set.test_array[0]))
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': {
'array_updates.1': [1, {
'test_array': [67, 67]
}]
}
}
}
data_set.sync_from_storage(-1)
self.assertListEqual([67, 67], list(data_set.test_array[1]))
def test_metadata_triggers_update(self):
name = 'Bobo'
user_data = PythonJsonStructure(data='plata', snapshot=False)
timestamp = datetime.datetime(2019, 12, 24)
default_array_name = 'ThatsAGoodName'
io_reader, io_writer = MemoryDataSetIOFactory.get_reader_writer_pair()
data_set_consumer = DataSet(storage_reader=io_reader)
data_set_producer = DataSet(storage_writer=io_writer)
data_set_producer.name = name
data_set_producer.user_data = user_data
data_set_producer.time_stamp = timestamp
data_set_producer.default_array_name = default_array_name
data_set_consumer.sync_from_storage(-1)
self.assertEqual(name, data_set_consumer.name)
self.assertDictEqual(user_data, data_set_consumer.user_data)
self.assertEqual(timestamp, data_set_consumer.time_stamp)
self.assertEqual(default_array_name,
data_set_consumer.default_array_name)
def test_sync_from_storage_array(self):
mock_queue = MagicMock()
with patch('qilib.data_set.mongo_data_set_io_reader.MongoDataSetIO') as mock_io, patch(
'qilib.data_set.mongo_data_set_io_reader.Thread') as thread, \
patch('qilib.data_set.mongo_data_set_io_reader.Queue', return_value=mock_queue):
mock_io.decode_numpy_array = MongoDataSetIO.decode_numpy_array
reader = MongoDataSetIOReader(name='test')
thread.assert_called_once()
mock_io.assert_called_once_with('test',
None,
create_if_not_found=False,
collection='data_sets',
database='qilib')
data_set = DataSet(storage_reader=reader)
set_array = DataArray(name='setter',
label='for_testing',
is_setpoint=True,
preset_data=np.array(range(0, 2)))
set_array[0] = 42
set_array[1] = 25
data_array = DataArray(name='test_array',
label='lab',
unit='V',
is_setpoint=False,
set_arrays=[set_array],
shape=(2, ))
update_data = {
"data_arrays": {
"setter": {
"name":
set_array.name,
"label":
set_array.label,
"unit":
set_array.unit,
"is_setpoint":
set_array.is_setpoint,
"set_arrays":
[array.name for array in set_array.set_arrays],
"preset_data":
MongoDataSetIO.encode_numpy_array(set_array)
}
}
}
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': update_data
}
}
data_set.sync_from_storage(-1)
update_data = {
"data_arrays.test_array": {
"name": data_array.name,
"label": data_array.label,
"unit": data_array.unit,
"is_setpoint": data_array.is_setpoint,
"set_arrays":
[array.name for array in data_array.set_arrays],
"preset_data":
MongoDataSetIO.encode_numpy_array(data_array)
}
}
mock_queue.get.return_value = {
'updateDescription': {
'updatedFields': update_data
}
}
data_set.sync_from_storage(-1)
self.assertEqual('test_array', data_set.test_array.name)
self.assertEqual('lab', data_set.test_array.label)
self.assertEqual('V', data_set.test_array.unit)
self.assertFalse(data_set.test_array.is_setpoint)
self.assertEqual('setter', data_array.set_arrays[0].name)
self.assertListEqual([42, 25], list(data_array.set_arrays[0]))
data_array[0] = 255
update_data["data_arrays.test_array"][
"preset_data"] = MongoDataSetIO.encode_numpy_array(data_array)
data_set.sync_from_storage(-1)
self.assertEqual(255, data_set.test_array[0])