def get_test_loader(raw_data, split_idx, args):
data_splits = data_utils.read_splits('%s/split_%d.txt' %
(args.data, split_idx))
test_loader = get_loader(raw_data,
data_splits['test'],
args,
shuffle=False,
num_workers=0,
batch_size=1)
return test_loader
def get_data_iterator(region_id, data_path, splits_path, data_split='test', collapse_time=True,
batch_size=32, shuffle=False, num_workers=0):
""" Creates an iterator for data in region 'region_id' for the days in `splits_path`
"""
params = cf.get_params(region_id=region_id, data_path=data_path, splits_path=splits_path)
params['data_params']['collapse_time'] = collapse_time
ds = create_dataset(data_split, params['data_params'])
dataloader = DataLoader(ds, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
data_splits, test_sequences = data_utils.read_splits(params['data_params']['train_splits'], params['data_params']['test_splits'])
test_dates = data_splits[data_splits.split=='test'].id_date.sort_values().values
return iter(dataloader), test_dates, params
def load_datasets(raw_data, split_idx, args):
data_splits = data_utils.read_splits('%s/split_%d.txt' %
(args.data, split_idx))
dataset_loaders = {}
if not args.test_mode:
dataset_loaders['train'] = get_loader(raw_data,
data_splits['train'],
args,
shuffle=True)
dataset_loaders['valid'] = get_loader(raw_data,
data_splits['valid'],
args,
shuffle=False)
dataset_loaders['test'] = get_loader(raw_data,
data_splits['test'],
args,
shuffle=False)
return dataset_loaders