def test_index_normalize_value(dtype, value, expected): index = ExplicitSecondaryIndex( column="col", dtype=dtype, index_storage_key="dataset_uuid/some_index.parquet" ) actual = index.normalize_value(index.dtype, value) assert actual == expected assert type(actual) == type(expected)
def test_serialization_normalization(key): """ Check that index normalizes values consistently after serializing. This is helpful to ensure correct behavior for cases such as when key=`datetime.datetime(2018, 1, 1, 12, 30)`, as this would be parsed to `pa.timestamp("us")` during index creation, but stored as `pa.timestamp("ns")`. """ index = ExplicitSecondaryIndex( column="col", index_dct={key: ["part_2", "part_4", "part_1"]}) index2 = pickle.loads(pickle.dumps(index)) assert index.normalize_value(index.dtype, key) == index2.normalize_value( index2.dtype, key)