def test_nest_empty(self):
data = DataArray()
self.assertEqual(data.shape, ())
mock_set_array = 'not really an array but we don\'t check'
mock_set_array2 = 'another one'
data.nest(2, action_index=44, set_array=mock_set_array)
data.nest(3, action_index=66, set_array=mock_set_array2)
# the array doesn't exist until you initialize it
self.assertIsNone(data.ndarray)
# but other attributes are set
self.assertEqual(data.shape, (3, 2))
self.assertEqual(data.action_indices, (66, 44))
self.assertEqual(data.set_arrays, (mock_set_array2, mock_set_array))
data.init_data()
self.assertEqual(data.ndarray.shape, (3, 2))
# after initializing data, you can't nest anymore because this isn't
# a preset array
with self.assertRaises(RuntimeError):
data.nest(4)
def test_init_data_error(self):
data = DataArray(preset_data=[1, 2])
data.shape = (3, )
# not sure when this would happen... but if you call init_data
# and it notices an inconsistency between shape and the actual
# data that's already there, it raises an error
with self.assertRaises(ValueError):
data.init_data()
def test_complete(self):
array = DataArray(name='y', shape=(5,))
array.init_data()
data = new_data(arrays=(array,), location=False)
self.syncing_array = array
self.sync_index = 0
data.sync = self.mock_sync
bf = DataSet.background_functions
bf['fail'] = self.failing_func
bf['log'] = self.logging_func
with LogCapture() as logs:
# grab info and warnings but not debug messages
logging.getLogger().setLevel(logging.INFO)
data.complete(delay=0.001)
logs = logs.value
expected_logs = [
'waiting for DataSet <False> to complete',
'DataSet: 0% complete',
'RuntimeError: it is called failing_func for a reason!',
'background at index 1',
'DataSet: 20% complete',
'RuntimeError: it is called failing_func for a reason!',
'background function fail failed twice in a row, removing it',
'background at index 2',
'DataSet: 40% complete',
'background at index 3',
'DataSet: 60% complete',
'background at index 4',
'DataSet: 80% complete',
'background at index 5',
'DataSet <False> is complete'
]
log_index = 0
for line in expected_logs:
self.assertIn(line, logs, logs)
try:
log_index_new = logs.index(line, log_index)
except ValueError:
raise ValueError('line {} not found after {} in: \n {}'.format(
line, log_index, logs))
self.assertTrue(log_index_new >= log_index, logs)
log_index = log_index_new + len(line) + 1 # +1 for \n
self.assertEqual(log_index, len(logs), logs)
def test_fraction_complete(self):
data = DataArray(shape=(5, 10))
self.assertIsNone(data.ndarray)
self.assertEqual(data.fraction_complete(), 0.0)
data.init_data()
self.assertEqual(data.fraction_complete(), 0.0)
# index = 1 * 10 + 7 - add 1 (for index 0) and you get 18
# each index is 2% of the total, so this is 36%
data[1, 7] = 1
self.assertEqual(data.fraction_complete(), 18 / 50)
# add a last_saved_index but modified_range is still bigger
data.mark_saved(13)
self.assertEqual(data.fraction_complete(), 18 / 50)
# now last_saved_index wins
data.mark_saved(19)
self.assertEqual(data.fraction_complete(), 20 / 50)
# now pretend we get more info from syncing
data.synced_index = 22
self.assertEqual(data.fraction_complete(), 23 / 50)
def test_preset_data(self):
onetwothree = [
# lists and tuples work
[1.0, 2.0, 3.0],
(1.0, 2.0, 3.0),
# iterators get automatically cast to floats
(i + 1 for i in range(3)),
map(float, range(1, 4)),
# and of course numpy arrays themselves work
np.array([1.0, 2.0, 3.0]),
]
expected123 = [1.0, 2.0, 3.0]
for item in onetwothree:
data = DataArray(preset_data=item)
self.assertEqual(data.ndarray.tolist(), expected123)
self.assertEqual(data.shape, (3, ))
# you can re-initialize a DataArray with the same shape data,
# but not with a different shape
list456 = [4, 5, 6]
data.init_data(data=list456)
self.assertEqual(data.ndarray.tolist(), list456)
with self.assertRaises(ValueError):
data.init_data([1, 2])
self.assertEqual(data.ndarray.tolist(), list456)
self.assertEqual(data.shape, (3, ))
# you can call init_data again with no data, and nothing changes
data.init_data()
self.assertEqual(data.ndarray.tolist(), list456)
self.assertEqual(data.shape, (3, ))
# multidimensional works too
list2d = [[1, 2], [3, 4]]
data2 = DataArray(preset_data=list2d)
self.assertEqual(data2.ndarray.tolist(), list2d)
self.assertEqual(data2.shape, (2, 2))
def read_one_file(self, data_set, f, ids_read):
# I have written more flexible code in my life
if len(data_set.arrays) != 0:
return
line, data_set.header = self._read_header(f, "# ")
# interpret metadata (header)
set_columns = []
set_arrays = ()
data_columns = []
data_arrays = []
c = [i + 1 for i in range(len(line.split('\t')))]
print(c)
for i in c:
if 'Column ' + str(i) not in data_set.header:
raise KeyError('File header does not match nr of columns '
'(Column' + str(i) + ')')
info = data_set.header['Column ' + str(i)]
if info['type'] == 'coordinate':
set_columns.append(['Column ' + str(i), i - 1])
elif info['type'] == 'value':
data_columns.append(['Column ' + str(i), i - 1])
set_columns.reverse()
shape = ()
for n in set_columns:
size = data_set.header[n[0]]['size']
shape += (int(size), )
name = data_set.header[n[0]]['name']
set_array = DataArray(label=name,
array_id="DataArray" + str(n[1]),
set_arrays=set_arrays,
shape=shape,
is_setpoint=True,
snapshot=None)
set_array.init_data()
data_set.add_array(set_array)
set_arrays = set_arrays + (set_array, )
for n in data_columns:
name = data_set.header[n[0]]['name']
data_array = DataArray(label=name,
array_id="DataArray" + str(n[1]),
set_arrays=set_arrays,
shape=shape,
snapshot=None)
data_array.init_data()
data_set.add_array(data_array)
data_arrays.append(data_array)
npoints = 1
print(shape)
for n in shape:
npoints = npoints * n
print(npoints)
indices = [0] * len(set_arrays)
elements = line.split('\t')
set_values = []
for i in set_columns:
set_values.append(float(elements[i[1]]))
p = 0
watchdog = 0
while True:
# print(indices)
# handle all elements in this line
if len(elements) != len(data_set.arrays):
raise Exception("Wrong number of columns in data row " +
str(p))
# Check hypercubic consistency
set_value_check = []
for i in set_columns:
set_value_check.append(float(elements[i[1]]))
for a, b in zip(set_value_check, set_values):
if a != b:
raise Exception("data is not hypercubic")
# add data points to the dataset
for c in data_columns:
i = tuple(indices)
data_set.arrays['DataArray' + str(c[1])][i] = float(
elements[c[1]])
# add set_points to the dataset
for j, c in enumerate(set_columns):
i = tuple(indices)[0:j + 1]
data_set.arrays['DataArray' + str(c[1])][i] = float(
elements[c[1]])
p += 1
if p >= npoints:
break
line = ""
watchdog = 0
while line is "" and watchdog < 100:
line = f.readline()
line = line.rstrip('\n\t\r')
watchdog += 1
if watchdog >= 100:
break
elements = line.split('\t')
# update indices en set new set_values to check hypercubicity
pointer = len(indices) - 1
indices[pointer] += 1
# new set value and update it to dataset
val = float(elements[set_columns[pointer][1]])
set_values[pointer] = val
while pointer >= 0 and indices[pointer] >= shape[pointer]:
# reset and decrease pointer
indices[pointer] = 0
pointer -= 1
indices[pointer] += 1
# new set value and update it to dataset
val = float(elements[set_columns[pointer][1]])
set_values[pointer] = val
if watchdog >= 100:
print("unexpected EOF")
print(p)
print(npoints)
line = f.readline()
line = line.rstrip('\n\t\r')
if line is not "":
raise Exception("File is supposed to be empty, but nonempty line "
"found (" + line + ")")
def _create_data_array(
self,
action_indices: Tuple[int],
result,
parameter: Parameter = None,
is_setpoint: bool = False,
name: str = None,
label: str = None,
unit: str = None,
):
"""Create a data array from a parameter and result.
The data array shape is extracted from the result shape, and the current
loop dimensions.
The data array is added to the current data set.
Args:
parameter: Parameter for which to create a DataArray. Can also be a
string, in which case it is the data_array name
result: Result returned by the Parameter
action_indices: Action indices for which to store parameter
is_setpoint: Whether the Parameter is used for sweeping or measuring
label: Data array label. If not provided, the parameter label is
used. If the parameter is a name string, the label is extracted
from the name.
unit: Data array unit. If not provided, the parameter unit is used.
Returns:
Newly created data array
"""
if parameter is None and name is None:
raise SyntaxError(
"When creating a data array, must provide either a parameter or a name"
)
if len(running_measurement().data_arrays) >= self.max_arrays:
raise RuntimeError(
f"Number of arrays in dataset exceeds "
f"Measurement.max_arrays={self.max_arrays}. Perhaps you forgot"
f"to encapsulate a loop with a Sweep()?"
)
array_kwargs = {
"is_setpoint": is_setpoint,
"action_indices": action_indices,
"shape": self.loop_shape,
}
if is_setpoint or isinstance(result, (np.ndarray, list)):
array_kwargs["shape"] += np.shape(result)
# Use dummy index (1, ) if measurement is performed outside a Sweep
if not array_kwargs["shape"]:
array_kwargs["shape"] = (1,)
if isinstance(parameter, Parameter):
array_kwargs["parameter"] = parameter
# Add a custom name
if name is not None:
array_kwargs["full_name"] = name
if label is not None:
array_kwargs["label"] = label
if unit is not None:
array_kwargs["unit"] = unit
else:
array_kwargs["name"] = name
if label is None:
label = name[0].capitalize() + name[1:].replace("_", " ")
array_kwargs["label"] = label
array_kwargs["unit"] = unit or ""
# Add setpoint arrays
if not is_setpoint:
array_kwargs["set_arrays"] = self._add_set_arrays(
action_indices, result, parameter=parameter, name=(name or parameter.name)
)
data_array = DataArray(**array_kwargs)
data_array.array_id = data_array.full_name
data_array.array_id += "_" + "_".join(str(k) for k in action_indices)
data_array.init_data()
self.dataset.add_array(data_array)
with self.timings.record(['dataset', 'save_metadata']):
self.dataset.save_metadata()
# Add array to set_arrays or to data_arrays of this Measurement
if is_setpoint:
self.set_arrays[action_indices] = data_array
else:
self.data_arrays[action_indices] = data_array
return data_array
