def test_serializer_accepts_one_dimensional_array(): s = numpy_to_record_serializer() array_data = [1.0, 2.0, 3.0] buf = s(np.array(array_data)) record_data = next(_read_recordio(buf)) record = Record() record.ParseFromString(record_data) assert record.features["values"].float64_tensor.values == array_data
def test_serializer(): s = numpy_to_record_serializer() array_data = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]] buf = s(np.array(array_data)) for record_data, expected in zip(_read_recordio(buf), array_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].float64_tensor.values == expected
def test_int_write_numpy_to_dense_tensor(): array_data = [[1, 2, 3], [10, 20, 3]] array = np.array(array_data) with tempfile.TemporaryFile() as f: write_numpy_to_dense_tensor(f, array) f.seek(0) for record_data, expected in zip(_read_recordio(f), array_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].int32_tensor.values == expected
def test_float32_write_numpy_to_dense_tensor(): array_data = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]] array = np.array(array_data).astype(np.dtype('float32')) with tempfile.TemporaryFile() as f: write_numpy_to_dense_tensor(f, array) f.seek(0) for record_data, expected in zip(_read_recordio(f), array_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].float32_tensor.values == expected
def test_float_label(): array_data = [[1, 2, 3], [10, 20, 3]] array = np.array(array_data) label_data = np.array([99, 98, 97]).astype(np.dtype('float64')) with tempfile.TemporaryFile() as f: write_numpy_to_dense_tensor(f, array, label_data) f.seek(0) for record_data, expected, label in zip(_read_recordio(f), array_data, label_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].int32_tensor.values == expected assert record.label["values"].float64_tensor.values == [label]
def test_float32_label(): array_data = [[1, 2, 3], [10, 20, 3]] array = np.array(array_data) label_data = np.array([99, 98, 97]).astype(np.dtype('float32')) with tempfile.TemporaryFile() as f: write_numpy_to_dense_tensor(f, array, label_data) f.seek(0) for record_data, expected, label in zip(_read_recordio(f), array_data, label_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].int32_tensor.values == expected assert record.label["values"].float32_tensor.values == [label]
def test_dense_int_write_spmatrix_to_sparse_tensor(): array_data = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]] keys_data = [[0, 1, 2], [0, 1, 2]] array = coo_matrix(np.array(array_data).astype(np.dtype('int'))) with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array) f.seek(0) for record_data, expected_data, expected_keys in zip(_read_recordio(f), array_data, keys_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].int32_tensor.values == expected_data assert record.features["values"].int32_tensor.keys == expected_keys assert record.features["values"].int32_tensor.shape == [len(expected_data)]
def test_dense_int_write_spmatrix_to_sparse_tensor(): array_data = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]] keys_data = [[0, 1, 2], [0, 1, 2]] array = coo_matrix(np.array(array_data).astype(np.dtype('int'))) with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array) f.seek(0) for record_data, expected_data, expected_keys in zip( _read_recordio(f), array_data, keys_data): record = Record() record.ParseFromString(record_data) assert record.features[ "values"].int32_tensor.values == expected_data assert record.features["values"].int32_tensor.keys == expected_keys assert record.features["values"].int32_tensor.shape == [ len(expected_data) ]
def test_dense_int_spmatrix_to_sparse_label(): array_data = [[1, 2, 3], [10, 20, 3]] keys_data = [[0, 1, 2], [0, 1, 2]] array = coo_matrix(np.array(array_data)) label_data = np.array([99, 98, 97]) with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array, label_data) f.seek(0) for record_data, expected_data, expected_keys, label in zip( _read_recordio(f), array_data, keys_data, label_data): record = Record() record.ParseFromString(record_data) assert record.features[ "values"].int32_tensor.values == expected_data assert record.features["values"].int32_tensor.keys == expected_keys assert record.label["values"].int32_tensor.values == [label] assert record.features["values"].int32_tensor.shape == [ len(expected_data) ]
def test_sparse_int_write_spmatrix_to_sparse_tensor(): n = 4 array_data = [[1.0, 2.0], [10.0, 30.0], [100.0, 200.0, 300.0, 400.0], [1000.0, 2000.0, 3000.0]] keys_data = [[0, 1], [1, 2], [0, 1, 2, 3], [0, 2, 3]] flatten_data = list(itertools.chain.from_iterable(array_data)) y_indices = list(itertools.chain.from_iterable(keys_data)) x_indices = [[i] * len(keys_data[i]) for i in range(len(keys_data))] x_indices = list(itertools.chain.from_iterable(x_indices)) array = coo_matrix((flatten_data, (x_indices, y_indices)), dtype='int') with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array) f.seek(0) for record_data, expected_data, expected_keys in zip(_read_recordio(f), array_data, keys_data): record = Record() record.ParseFromString(record_data) assert record.features["values"].int32_tensor.values == expected_data assert record.features["values"].int32_tensor.keys == expected_keys assert record.features["values"].int32_tensor.shape == [n]
def test_dense_float64_spmatrix_to_sparse_label(): array_data = [[1, 2, 3], [10, 20, 3]] keys_data = [[0, 1, 2], [0, 1, 2]] array = coo_matrix(np.array(array_data).astype('float64')) label_data = np.array([99, 98, 97]) with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array, label_data) f.seek(0) for record_data, expected_data, expected_keys, label in zip( _read_recordio(f), array_data, keys_data, label_data ): record = Record() record.ParseFromString(record_data) assert record.features["values"].float64_tensor.values == expected_data assert record.features["values"].float64_tensor.keys == expected_keys assert record.label["values"].int32_tensor.values == [label] assert record.features["values"].float64_tensor.shape == [len(expected_data)]
def test_sparse_int_write_spmatrix_to_sparse_tensor(): n = 4 array_data = [[1.0, 2.0], [10.0, 30.0], [100.0, 200.0, 300.0, 400.0], [1000.0, 2000.0, 3000.0]] keys_data = [[0, 1], [1, 2], [0, 1, 2, 3], [0, 2, 3]] flatten_data = list(itertools.chain.from_iterable(array_data)) y_indices = list(itertools.chain.from_iterable(keys_data)) x_indices = [[i] * len(keys_data[i]) for i in range(len(keys_data))] x_indices = list(itertools.chain.from_iterable(x_indices)) array = coo_matrix((flatten_data, (x_indices, y_indices)), dtype='int') with tempfile.TemporaryFile() as f: write_spmatrix_to_sparse_tensor(f, array) f.seek(0) for record_data, expected_data, expected_keys in zip( _read_recordio(f), array_data, keys_data): record = Record() record.ParseFromString(record_data) assert record.features[ "values"].int32_tensor.values == expected_data assert record.features["values"].int32_tensor.keys == expected_keys assert record.features["values"].int32_tensor.shape == [n]