def get_dataset(path, test_ratio, batch_size):
raw_data = pd.read_csv(path)
x_data = raw_data.iloc[:, 1:-1]
y_data = raw_data.iloc[:, -1]
x_train, x_val, y_train, y_val = pre_process(x_data, y_data, test_ratio)
x_train = np.expand_dims(x_train, axis=-1)
y_train = np.expand_dims(y_train, axis=1)
x_val = np.expand_dims(x_val, axis=-1)
y_val = np.expand_dims(y_val, axis=1)
print(x_train.shape)
print(y_train.shape)
print(x_val.shape)
print(y_val.shape)
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
validation_dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))
def reshape(x, y):
return x, y
train__dataset = train_dataset.map(reshape).cache().shuffle(32).repeat(
1).batch(batch_size)
validation_dataset = validation_dataset.map(reshape).cache().shuffle(
32).repeat(1).batch(batch_size)
print(train__dataset)
print(validation_dataset)
return train_dataset, validation_dataset
def get_dataset(path, test_ratio):
raw_data = pd.read_csv(path)
x_data = raw_data.iloc[:, 1:-1]
y_data = raw_data.iloc[:, -1]
x_train, x_val, y_train, y_val, x_test = pre_process(
x_data, y_data, test_ratio)
return x_train, x_val, y_train, y_val
def get_dataset(path, eval_ratio):
raw_data = pd.read_csv(path)
x_data = raw_data.iloc[:, 1:-1]
y_data = raw_data.iloc[:, -1]
x_train, x_val, y_train, y_val, x_test = pre_process(x_data, y_data, 0.1)
print(x_test.shape)
if eval_ratio == 0:
x_data = np.concatenate((x_train, x_val), axis=0)
y_data = np.concatenate((y_train, y_val), axis=0)
return x_data, y_data
return x_train, x_val, y_train, y_val
def get_dataset(val_ratio,for_model):
path = "./data/train.csv"
path2 = "./data/test.csv"
raw_data1 = pd.read_csv(path)
print(raw_data1.shape)
raw_data2 = pd.read_csv(path2)
print(raw_data2.shape)
raw_data = pd.concat((raw_data1,raw_data2),axis=0)
print(f"Raw data {raw_data.shape}")
x_data = raw_data.iloc[:, 1:-1]
y_data = raw_data1.iloc[:, -1]
x_train, x_val, y_train, y_val,x_test = pre_process(x_data, y_data, val_ratio,for_model)
return x_train, x_val, y_train, y_val,x_test