def read_data(
reader: DatasetReader,
train_data_path:
str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv",
valid_data_path:
str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv"
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
logger.critical("Reading the data. Lazy variable set to {}".format(
reader.lazy))
start_time = time.time()
'''Expect: that this is the only time it is called'''
reader.mode = "train"
training_data = reader.read(train_data_path)
# instead, we will set the examples differently here
reader.mode = "valid"
validation_data = reader.read(
valid_data_path) #need to unlimit the examples here...
logger.critical(
"Finished the call to read the data. Time took {}".format(time.time() -
start_time))
return training_data, validation_data
