def parameter_test_helper(ds: DataSet, toplevel_names: Sequence[str], expected_names: Dict[str, Sequence[str]], expected_shapes: Dict[str, Sequence[Tuple[int, ...]]], expected_values: Dict[str, Sequence[np.ndarray]], start: Optional[int] = None, end: Optional[int] = None): """ A helper function to compare the data we actually read out of a given dataset with the expected data. Args: ds: the dataset in question toplevel_names: names of the toplevel parameters of the dataset expected_names: names of the parameters expected to be loaded for a given parameter as a sequence indexed by the parameter. expected_shapes: expected shapes of the parameters loaded. The shapes should be stored as a tuple per parameter in a sequence containing all the loaded parameters for a given requested parameter. expected_values: expected content of the data arrays stored in a sequenceexpected_names: """ data = ds.get_parameter_data(*toplevel_names, start=start, end=end) dataframe = ds.get_data_as_pandas_dataframe(*toplevel_names, start=start, end=end) all_data = ds.get_parameter_data(start=start, end=end) all_dataframe = ds.get_data_as_pandas_dataframe(start=start, end=end) all_parameters = list(all_data.keys()) assert set(data.keys()).issubset(set(all_parameters)) assert list(data.keys()) == list(dataframe.keys()) assert len(data.keys()) == len(toplevel_names) assert len(dataframe.keys()) == len(toplevel_names) verify_data_dict(data, dataframe, toplevel_names, expected_names, expected_shapes, expected_values) verify_data_dict(all_data, all_dataframe, toplevel_names, expected_names, expected_shapes, expected_values) # Now lets remove a random element from the list # We do this one by one until there is only one element in the list subset_names = copy(all_parameters) while len(subset_names) > 1: elem_to_remove = random.randint(0, len(subset_names) - 1) name_removed = subset_names.pop(elem_to_remove) expected_names.pop(name_removed) expected_shapes.pop(name_removed) expected_values.pop(name_removed) subset_data = ds.get_parameter_data(*subset_names, start=start, end=end) subset_dataframe = ds.get_data_as_pandas_dataframe(*subset_names, start=start, end=end) verify_data_dict(subset_data, subset_dataframe, subset_names, expected_names, expected_shapes, expected_values)
def parameter_test_helper(ds, toplevel_names, expected_names, expected_shapes, expected_values, start=None, end=None): data = ds.get_parameter_data(*toplevel_names, start=start, end=end) dataframe = ds.get_data_as_pandas_dataframe(*toplevel_names, start=start, end=end) all_data = ds.get_parameter_data(start=start, end=end) all_dataframe = ds.get_data_as_pandas_dataframe(start=start, end=end) all_parameters = list(all_data.keys()) assert set(data.keys()).issubset(set(all_parameters)) assert list(data.keys()) == list(dataframe.keys()) assert len(data.keys()) == len(toplevel_names) assert len(dataframe.keys()) == len(toplevel_names) verify_data_dict(data, dataframe, toplevel_names, expected_names, expected_shapes, expected_values) verify_data_dict(all_data, all_dataframe, toplevel_names, expected_names, expected_shapes, expected_values) # Now lets remove a random element from the list # We do this one by one until there is only one element in the list subset_names = copy(all_parameters) while len(subset_names) > 1: elem_to_remove = random.randint(0, len(subset_names) - 1) name_removed = subset_names.pop(elem_to_remove) expected_names.pop(name_removed) expected_shapes.pop(name_removed) expected_values.pop(name_removed) subset_data = ds.get_parameter_data(*subset_names, start=start, end=end) subset_dataframe = ds.get_data_as_pandas_dataframe(*subset_names, start=start, end=end) verify_data_dict(subset_data, subset_dataframe, subset_names, expected_names, expected_shapes, expected_values)