def parameter_test_helper(ds: DataSet,
toplevel_names: Sequence[str],
expected_names: Dict[str, Sequence[str]],
expected_shapes: Dict[str, Sequence[Tuple[int, ...]]],
expected_values: Dict[str, Sequence[np.ndarray]],
start: Optional[int] = None,
end: Optional[int] = None):
"""
A helper function to compare the data we actually read out of a given
dataset with the expected data.
Args:
ds: the dataset in question
toplevel_names: names of the toplevel parameters of the dataset
expected_names: names of the parameters expected to be loaded for a
given parameter as a sequence indexed by the parameter.
expected_shapes: expected shapes of the parameters loaded. The shapes
should be stored as a tuple per parameter in a sequence containing
all the loaded parameters for a given requested parameter.
expected_values: expected content of the data arrays stored in a
sequenceexpected_names:
"""
data = ds.get_parameter_data(*toplevel_names, start=start, end=end)
dataframe = ds.get_data_as_pandas_dataframe(*toplevel_names,
start=start,
end=end)
all_data = ds.get_parameter_data(start=start, end=end)
all_dataframe = ds.get_data_as_pandas_dataframe(start=start, end=end)
all_parameters = list(all_data.keys())
assert set(data.keys()).issubset(set(all_parameters))
assert list(data.keys()) == list(dataframe.keys())
assert len(data.keys()) == len(toplevel_names)
assert len(dataframe.keys()) == len(toplevel_names)
verify_data_dict(data, dataframe, toplevel_names, expected_names,
expected_shapes, expected_values)
verify_data_dict(all_data, all_dataframe, toplevel_names, expected_names,
expected_shapes, expected_values)
# Now lets remove a random element from the list
# We do this one by one until there is only one element in the list
subset_names = copy(all_parameters)
while len(subset_names) > 1:
elem_to_remove = random.randint(0, len(subset_names) - 1)
name_removed = subset_names.pop(elem_to_remove)
expected_names.pop(name_removed)
expected_shapes.pop(name_removed)
expected_values.pop(name_removed)
subset_data = ds.get_parameter_data(*subset_names,
start=start, end=end)
subset_dataframe = ds.get_data_as_pandas_dataframe(*subset_names,
start=start,
end=end)
verify_data_dict(subset_data, subset_dataframe, subset_names,
expected_names, expected_shapes, expected_values)
def parameter_test_helper(ds,
toplevel_names,
expected_names,
expected_shapes,
expected_values,
start=None,
end=None):
data = ds.get_parameter_data(*toplevel_names, start=start, end=end)
dataframe = ds.get_data_as_pandas_dataframe(*toplevel_names,
start=start,
end=end)
all_data = ds.get_parameter_data(start=start, end=end)
all_dataframe = ds.get_data_as_pandas_dataframe(start=start, end=end)
all_parameters = list(all_data.keys())
assert set(data.keys()).issubset(set(all_parameters))
assert list(data.keys()) == list(dataframe.keys())
assert len(data.keys()) == len(toplevel_names)
assert len(dataframe.keys()) == len(toplevel_names)
verify_data_dict(data, dataframe, toplevel_names, expected_names,
expected_shapes, expected_values)
verify_data_dict(all_data, all_dataframe, toplevel_names, expected_names,
expected_shapes, expected_values)
# Now lets remove a random element from the list
# We do this one by one until there is only one element in the list
subset_names = copy(all_parameters)
while len(subset_names) > 1:
elem_to_remove = random.randint(0, len(subset_names) - 1)
name_removed = subset_names.pop(elem_to_remove)
expected_names.pop(name_removed)
expected_shapes.pop(name_removed)
expected_values.pop(name_removed)
subset_data = ds.get_parameter_data(*subset_names,
start=start,
end=end)
subset_dataframe = ds.get_data_as_pandas_dataframe(*subset_names,
start=start,
end=end)
verify_data_dict(subset_data, subset_dataframe, subset_names,
expected_names, expected_shapes, expected_values)
