def create_data_loader(X, y=None, batch_size=1):
if isinstance(X, pd.DataFrame):
X = X.to_numpy()
if isinstance(y, pd.Series):
y = y.to_numpy()
if isinstance(X, np.ndarray):
dataset = NumpyTorchDataset(X, y)
if isinstance(X, str): #Assume that this is path to hdf5 file
dataset = HDF5TorchDataset(X)
return DataLoader(dataset, batch_size=batch_size)
def create_data_loader(X, y=None, batch_size=1):
import torch
from torch.utils.data import DataLoader, TensorDataset
from lale.util.batch_data_dictionary_dataset import BatchDataDict
from lale.util.hdf5_to_torch_dataset import HDF5TorchDataset
from lale.util.numpy_to_torch_dataset import NumpyTorchDataset
if isinstance(X, pd.DataFrame):
X = X.to_numpy()
if isinstance(y, pd.Series):
y = y.to_numpy()
dataset = NumpyTorchDataset(X, y)
elif isinstance(X, scipy.sparse.csr.csr_matrix):
# unfortunately, NumpyTorchDataset won't accept a subclass of np.ndarray
X = X.toarray()
if isinstance(y, lale.datasets.data_schemas.NDArrayWithSchema):
y = y.view(np.ndarray)
dataset = NumpyTorchDataset(X, y)
elif isinstance(X, np.ndarray):
# unfortunately, NumpyTorchDataset won't accept a subclass of np.ndarray
if isinstance(X, lale.datasets.data_schemas.NDArrayWithSchema):
X = X.view(np.ndarray)
if isinstance(y, lale.datasets.data_schemas.NDArrayWithSchema):
y = y.view(np.ndarray)
dataset = NumpyTorchDataset(X, y)
elif isinstance(X, str): # Assume that this is path to hdf5 file
dataset = HDF5TorchDataset(X)
elif isinstance(X, BatchDataDict):
dataset = X
def my_collate_fn(batch):
return batch[
0
] # because BatchDataDict's get_item returns a batch, so no collate is required.
return DataLoader(dataset, batch_size=1, collate_fn=my_collate_fn)
elif isinstance(X, dict): # Assumed that it is data indexed by batch number
return [X]
elif isinstance(X, torch.Tensor) and y is not None:
if isinstance(y, np.ndarray):
y = torch.from_numpy(y)
dataset = TensorDataset(X, y)
elif isinstance(X, torch.Tensor):
dataset = TensorDataset(X)
else:
raise TypeError(
"Can not create a data loader for a dataset with type {}".format(type(X))
)
return DataLoader(dataset, batch_size=batch_size)
def create_data_loader(X, y = None, batch_size = 1):
if isinstance(X, pd.DataFrame):
X = X.to_numpy()
if isinstance(y, pd.Series):
y = y.to_numpy()
elif isinstance(X, np.ndarray):
dataset = NumpyTorchDataset(X, y)
elif isinstance(X, str):#Assume that this is path to hdf5 file
dataset = HDF5TorchDataset(X)
elif isinstance(X, dict): #Assumed that it is data indexed by batch number
#dataset = BatchDataDictDataset(X)
return X.values()
elif isinstance(X, torch.Tensor) and y is not None:
dataset = TensorDataset(X, y)
elif isinstance(X, torch.Tensor):
dataset = TensorDataset(X)
else:
raise TypeError("Can not create a data loader for a dataset with type {}".format(type(X)))
return DataLoader(dataset, batch_size=batch_size)
def create_data_loader(X, y=None, batch_size=1, num_workers=0):
"""A function that takes a dataset as input and outputs a Pytorch dataloader.
Parameters
----------
X : Input data.
The formats supported are Pandas DataFrame, Numpy array,
a sparse matrix, torch.tensor, torch.utils.data.Dataset, path to a HDF5 file,
lale.util.batch_data_dictionary_dataset.BatchDataDict,
a Python dictionary of the format `{"dataset": torch.utils.data.Dataset,
"collate_fn":collate_fn for torch.utils.data.DataLoader}`
y : Labels., optional
Supported formats are Numpy array or Pandas series, by default None
batch_size : int, optional
Number of samples in each batch, by default 1
num_workers : int, optional
Number of workers used by the data loader, by default 0
Returns
-------
torch.utils.data.DataLoader
Raises
------
TypeError
Raises a TypeError if the input format is not supported.
"""
import torch
from torch.utils.data import DataLoader, Dataset, TensorDataset
from lale.util.batch_data_dictionary_dataset import BatchDataDict
from lale.util.hdf5_to_torch_dataset import HDF5TorchDataset
from lale.util.numpy_to_torch_dataset import NumpyTorchDataset
collate_fn = None
worker_init_fn = None
def numpy_collate_fn(batch):
return_X = None
return_y = None
for item in batch:
if isinstance(item, tuple):
if return_X is None:
return_X = item[0]
else:
return_X = np.vstack((return_X, item[0]))
if return_y is None:
return_y = item[1]
else:
return_y = np.vstack((return_y, item[1]))
else:
if return_X is None:
return_X = item
else:
return_X = np.vstack((return_X, item))
if return_y is not None:
if len(return_y.shape) > 1 and return_y.shape[1] == 1:
return_y = np.reshape(return_y, (len(return_y), ))
return return_X, return_y
else:
return return_X
if isinstance(X, Dataset):
dataset = X
elif isinstance(X, pd.DataFrame):
X = X.to_numpy()
if isinstance(y, pd.Series):
y = y.to_numpy()
dataset = NumpyTorchDataset(X, y)
collate_fn = numpy_collate_fn
elif isinstance(X, scipy.sparse.csr.csr_matrix):
# unfortunately, NumpyTorchDataset won't accept a subclass of np.ndarray
X = X.toarray()
if isinstance(y, lale.datasets.data_schemas.NDArrayWithSchema):
y = y.view(np.ndarray)
dataset = NumpyTorchDataset(X, y)
collate_fn = numpy_collate_fn
elif isinstance(X, np.ndarray):
# unfortunately, NumpyTorchDataset won't accept a subclass of np.ndarray
if isinstance(X, lale.datasets.data_schemas.NDArrayWithSchema):
X = X.view(np.ndarray)
if isinstance(y, lale.datasets.data_schemas.NDArrayWithSchema):
y = y.view(np.ndarray)
dataset = NumpyTorchDataset(X, y)
collate_fn = numpy_collate_fn
elif isinstance(X, str): # Assume that this is path to hdf5 file
dataset = HDF5TorchDataset(X)
elif isinstance(X, BatchDataDict):
dataset = X
def my_collate_fn(batch):
return batch[
0] # because BatchDataDict's get_item returns a batch, so no collate is required.
return DataLoader(dataset, batch_size=1, collate_fn=my_collate_fn)
elif isinstance(X,
dict): # Assumed that it is data indexed by batch number
if "dataset" in X:
dataset = X["dataset"]
collate_fn = X.get("collate_fn", None)
worker_init_fn = getattr(dataset, "worker_init_fn", None)
else:
return [X]
elif isinstance(X, torch.Tensor) and y is not None:
if isinstance(y, np.ndarray):
y = torch.from_numpy(y)
dataset = TensorDataset(X, y)
elif isinstance(X, torch.Tensor):
dataset = TensorDataset(X)
else:
raise TypeError(
"Can not create a data loader for a dataset with type {}".format(
type(X)))
return DataLoader(
dataset,
batch_size=batch_size,
collate_fn=collate_fn,
num_workers=num_workers,
worker_init_fn=worker_init_fn,
)