def get_dataset(data_pars=None, out_pars=None, **kwargs):
device = _get_device()
dataset = data_pars['data_info'].get('dataset', None)
batch_size = data_pars['data_info'].get('batch_size', 64)
from mlmodels.dataloader import DataLoader
loader = DataLoader(data_pars)
if dataset:
loader.compute()
try:
dataset, internal_states = loader.get_data()
trainset, validset, vocab = dataset
train_iter, valid_iter = create_data_iterator(
batch_size, trainset, validset, device)
except:
raise Exception(
"the last Preprocessor have to return (trainset, validset, vocab), internal_states."
)
return train_iter, valid_iter, vocab
else:
raise Exception("dataset not provided ")
return 0
def get_dataset(_model, preprocessor, _preprocessor_pars, data_pars):
from mlmodels.dataloader import DataLoader
dataset = data_pars['data_info'].get('dataset', None)
loader = DataLoader(data_pars)
if dataset:
loader.compute()
try:
dataset, internal_states = loader.get_data()
trainset, validset = dataset
trainloader = get_data_loader(_model, preprocessor,
_preprocessor_pars["train"],
trainset)
testloader = get_data_loader(_model, preprocessor,
_preprocessor_pars["test"], validset)
except:
raise Exception(
"the last Preprocessor have to return (trainset, validset), internal_states."
)
return trainloader, testloader
else:
raise Exception("Please add dataset in datainfo")
return 0
def get_dataset(data_pars=None, **kw):
"""
JSON data_pars to get dataset
"data_pars": { "data_path": "dataset/GOOG-year.csv", "data_type": "pandas",
"size": [0, 0, 6], "output_size": [0, 6] },
"""
from mlmodels.dataloader import DataLoader
loader = DataLoader(data_pars)
loader.compute()
return loader.get_data()
def get_dataset(data_pars=None, **kw):
"""
JSON data_pars to get dataset
"data_pars": { "data_path": "dataset/GOOG-year.csv", "data_type": "pandas",
"size": [0, 0, 6], "output_size": [0, 6] },
"""
print('Loading data...')
maxlen = data_pars['data_info']['maxlen']
loader = DataLoader(data_pars)
loader.compute()
dataset, internal_states = loader.get_data()
# return dataset
Xtrain, ytrain, Xtest, ytest = dataset
Xtrain = sequence.pad_sequences(Xtrain, maxlen=maxlen)
Xtest = sequence.pad_sequences(Xtest, maxlen=maxlen)
return Xtrain, Xtest, ytrain, ytest
def get_dataset(data_pars=None, **kw):
#if data_pars['dataset'] == 'MNIST':
# train_loader, valid_loader = get_dataset_mnist_torch(data_pars)
# return train_loader, valid_loader
from mlmodels.dataloader import DataLoader
loader = DataLoader(data_pars)
if data_pars['data_info']['dataset'] :
loader.compute()
try:
(train_loader, valid_loader), internal_states = loader.get_data()
except:
raise Exception("the last Preprocessor have to return (train_loader, valid_loader), internal_states.")
return train_loader, valid_loader
else:
raise Exception("dataset not provided ")
return 0
def get_dataset(data_pars):
loader = DataLoader(data_pars)
loader.compute()
return loader.get_data()