def create_data_loader(data, subset=False):
# Build readers, discretizers, normalizers
print("Creating Data File Reader")
#Using val_test set for visualization
data_reader = Reader(dataset_dir=os.path.join(data, 'test'),
listfile=os.path.join(data, 'test', 'listfile.csv'),
period_length=24.0)
#For Hourly Task we need to limit the amount of data for visualization
if subset:
print("limiting data")
data_reader.limit_data(10)
print("Initializing Discretizer and Normalizer")
discretizer = DiscretizerContinuous(timestep=1.0,
store_masks=False,
impute_strategy='previous',
start_time='zero')
discretizer_header = discretizer.transform(
data_reader.read_example(0)["X"])[1]
cont_channels = [
i for (i, x) in enumerate(discretizer_header) if x.find("->") == -1
]
normalizer = Normalizer(
fields=cont_channels) # choose here which columns to standardize
normalizer_state = args.normalizer_state
if normalizer_state is None:
normalizer_state = 'ptemb_ts{}.input_str:{}.start_time:zero.normalizer'.format(
args.timestep, args.imputation)
normalizer_state = os.path.join(os.path.dirname(__file__),
normalizer_state)
normalizer.load_params(normalizer_state)
#Create Dataset + DataLoader
print("Building Dataset")
data_dataset = ClassDataset(reader=data_reader,
discretizer=discretizer,
normalizer=normalizer,
return_name=False,
embed_method=args.embed_method)
print("Building DataLoader")
data_loader = DataLoader(data_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
return data_loader
listfile=os.path.join(args.data, 'val',
'listfile.csv'),
period_length=24.0)
val_reader = DayReader(dataset_dir=os.path.join(args.data, 'val_test'),
listfile=os.path.join(args.data, 'val_test',
'listfile.csv'),
period_length=24.0)
#Limit the Percent of Data to use for training
if args.percent_data != 100:
train_reader.limit_data(args.percent_data)
print("Initializing Discretizer and Normalizer")
discretizer = DiscretizerContinuous(timestep=1.0,
store_masks=False,
impute_strategy='previous',
start_time='zero')
discretizer_header = discretizer.transform(
train_reader.read_example(0)["X"])[1]
cont_channels = [
i for (i, x) in enumerate(discretizer_header) if x.find("->") == -1
]
normalizer = Normalizer(
fields=cont_channels) # choose here which columns to standardize
normalizer_state = args.normalizer_state
if normalizer_state is None:
normalizer_state = 'ptemb_ts{}.input_str:{}.start_time:zero.normalizer'.format(
args.timestep, args.imputation)
normalizer_state = os.path.join(os.path.dirname(__file__),
def main():
parser = argparse.ArgumentParser(description='Script for creating a normalizer state - a file which stores the '
'means and standard deviations of columns of the output of a '
'discretizer, which are later used to standardize the input of '
'neural models.')
parser.add_argument('--task', type=str, required=True,
choices=['ihm', 'decomp', 'los', 'pheno', 'multi', 'ptemb'])
parser.add_argument('--timestep', type=float, default=1.0,
help="Rate of the re-sampling to discretize time-series.")
parser.add_argument('--impute_strategy', type=str, default='previous',
choices=['zero', 'next', 'previous', 'normal_value'],
help='Strategy for imputing missing values.')
parser.add_argument('--start_time', type=str, choices=['zero', 'relative'],
help='Specifies the start time of discretization. Zero means to use the beginning of '
'the ICU stay. Relative means to use the time of the first ICU event')
parser.add_argument('--store_masks', dest='store_masks', action='store_true',
help='Store masks that specify observed/imputed values.')
parser.add_argument('--no-masks', dest='store_masks', action='store_false',
help='Do not store that specify specifying observed/imputed values.')
parser.add_argument('--n_samples', type=int, default=-1, help='How many samples to use to estimates means and '
'standard deviations. Set -1 to use all training samples.')
parser.add_argument('--output_dir', type=str, help='Directory where the output file will be saved.',
default='.')
parser.add_argument('--data', type=str, required=True, help='Path to the task data.')
parser.set_defaults(store_masks=True)
args = parser.parse_args()
print(args)
# create the reader
reader = None
dataset_dir = os.path.join(args.data, 'train')
if args.task == 'ihm':
reader = InHospitalMortalityReader(dataset_dir=dataset_dir, period_length=48.0)
if args.task == 'decomp':
reader = DecompensationReader(dataset_dir=dataset_dir)
if args.task == 'los':
reader = LengthOfStayReader(dataset_dir=dataset_dir)
if args.task == 'pheno':
reader = PhenotypingReader(dataset_dir=dataset_dir)
if args.task == 'multi':
reader = MultitaskReader(dataset_dir=dataset_dir)
if args.task == 'ptemb':
reader = PatientEmbeddingReader(dataset_dir=dataset_dir)
# create the discretizer
discretizer = DiscretizerContinuous(timestep=args.timestep,
store_masks=args.store_masks,
impute_strategy=args.impute_strategy,
start_time=args.start_time)
discretizer_header = reader.read_example(0)['header']
continuous_channels = [i for (i, x) in enumerate(discretizer_header) if x.find("->") == -1]
# create the normalizer
normalizer = Normalizer(fields=continuous_channels)
# read all examples and store the state of the normalizer
n_samples = args.n_samples
if n_samples == -1:
n_samples = reader.get_number_of_examples()
for i in range(n_samples):
if i % 1000 == 0:
print('Processed {} / {} samples'.format(i, n_samples), end='\r')
ret = reader.read_example(i)
data, new_header = discretizer.transform(ret['X'], end=ret['t'])
normalizer._feed_data(data)
print('\n')
file_name = '{}_ts:{:.2f}_impute:{}_start:{}_masks:{}_n:{}.normalizer'.format(
args.task, args.timestep, args.impute_strategy, args.start_time, args.store_masks, n_samples)
file_name = os.path.join(args.output_dir, file_name)
print('Saving the state in {} ...'.format(file_name))
normalizer._save_params(file_name)