def __set_dataset2(self, train, test):
m_list = list(train.keys())
RMT = ReshapeMergerTree()
##Make train/test_input/output arrays in each m_list.
train_input_dict, train_output_dict = {}, {}
test_input_dict, test_output_dict = {}, {}
for m_key in m_list:
train_input_dict[m_key], train_output_dict[
m_key] = RMT.make_dataset(train[m_key], self.input_size,
self.output_size)
test_input_dict[m_key], test_output_dict[m_key] = RMT.make_dataset(
test[m_key], self.input_size, self.output_size)
if m_key == m_list[0]:
train_input, train_output = train_input_dict[
m_key], train_output_dict[m_key]
else:
train_input, train_output = np.concatenate(
[train_input, train_input_dict[m_key]],
axis=0), np.concatenate(
[train_output, train_output_dict[m_key]], axis=0)
##Normalize these input/output arrays.
##The dict-array is normalized by train-array's normalization parameters.
Norm_input, Norm_output = Normalization(
self.norm_format), Normalization(self.norm_format)
train_input, train_output = Norm_input.run(
train_input), Norm_output.run(train_output)
for m_key in m_list:
train_input_dict[m_key], train_output_dict[
m_key] = Norm_input.run_predict(
train_input_dict[m_key]), Norm_output.run_predict(
train_output_dict[m_key])
test_input_dict[m_key], test_output_dict[
m_key] = Norm_input.run_predict(
test_input_dict[m_key]), Norm_output.run_predict(
test_output_dict[m_key])
self.Norm_input, self.Norm_output = Norm_input, Norm_output
##Masking process to prevent division by zero.
mask = (train_output == 0.0)
train_output[mask] += 1e-7
for m_key in m_list:
mask = (train_output_dict[m_key] == 0.0)
train_output_dict[m_key][mask] += 1e-7
mask = (test_output_dict[m_key] == 0.0)
test_output_dict[m_key][mask] += 1e-7
return train_input, train_output, train_input_dict, train_output_dict, test_input_dict, test_output_dict
def set_dataset(self, dataset, train_ratio, fft_format, norm_format):
mlist = dataset.keys()
MTTD = MakeTrainTestDataset(mlist)
train, test = MTTD.split(dataset, train_ratio)
RMT_train, RMT_test = {}, {}
train_input, train_output = {}, {}
test_input, test_output = {}, {}
train_input_, train_output_ = None, None
test_input_, test_output_ = None, None
if fft_format == "fft":
fft = lambda x: np.fft.fft(x)
elif fft_format == "rfft":
fft = lambda x: np.fft.rfft(x)
if self.is_epoch_in_each_mlist:
Norm_train_input, Norm_train_output = {}, {}
Norm_test_input, Norm_test_output = {}, {}
for m_key in mlist:
Norm_train_input[m_key] = Normalization(norm_format)
Norm_train_output[m_key] = Normalization(norm_format)
Norm_test_input[m_key] = Normalization(norm_format)
Norm_test_output[m_key] = Normalization(norm_format)
Norm_train_input_ = Normalization(norm_format)
Norm_train_output_ = Normalization(norm_format)
Norm_test_input_ = Normalization(norm_format)
Norm_test_output_ = Normalization(norm_format)
for m_key in mlist:
RMT_train[m_key] = ReshapeMergerTree()
RMT_test[m_key] = ReshapeMergerTree()
train_input[m_key], train_output[m_key] = RMT_train[
m_key].make_dataset(train[m_key], self.input_size,
self.output_size)
test_input[m_key], test_output[m_key] = RMT_test[
m_key].make_dataset(test[m_key], self.input_size,
self.output_size)
if train_input_ is None and test_input_ is None:
train_input_, train_output_ = train_input[m_key], train_output[
m_key]
test_input_, test_output_ = test_input[m_key], test_output[
m_key]
else:
train_input_ = np.concatenate(
[train_input_, train_input[m_key]], axis=0)
train_output_ = np.concatenate(
[train_output_, train_output[m_key]], axis=0)
test_input_ = np.concatenate([test_input_, test_input[m_key]],
axis=0)
test_output_ = np.concatenate(
[test_output_, test_output[m_key]], axis=0)
if self.is_epoch_in_each_mlist:
train_input[m_key] = Norm_train_input[m_key].run(
fft(train_input[m_key]))
train_output[m_key] = Norm_train_output[m_key].run(
fft(train_output[m_key]))
test_input[m_key] = Norm_test_input[m_key].run(
fft(test_input[m_key]))
test_output[m_key] = Norm_test_output[m_key].run(
fft(test_output[m_key]))
train_input_ = Norm_train_input_.run(fft(train_input_))
train_output_ = Norm_train_output_.run(fft(train_output_))
test_input_ = Norm_test_input_.run(fft(test_input_))
test_output_ = Norm_test_output_.run(fft(test_output_))
train_mask_real = (train_output_.real == 0.0)
train_mask_imag = (train_output_.imag == 0.0)
train_output_[train_mask_real] += 1e-7
train_output_[train_mask_imag] += 1e-7j
if self.is_epoch_in_each_mlist:
for m_key in mlist:
train_mask_real = (train_output[m_key].real == 0.0)
train_mask_imag = (train_output[m_key].imag == 0.0)
test_mask_real = (test_output[m_key].real == 0.0)
test_mask_imag = (test_output[m_key].imag == 0.0)
train_output[m_key][train_mask_real] += 1e-7
train_output[m_key][train_mask_imag] += 1e-7j
test_output[m_key][test_mask_real] += 1e-7
test_output[m_key][test_mask_imag] += 1e-7j
return train_input_, train_output_, train_input, train_output, test_input, test_output
else:
test_mask_real = (test_output_.real == 0.0)
test_mask_imag = (test_output_.imag == 0.0)
test_output_[test_mask_real] += 1e-7
test_output_[test_mask_imag] += 1e-7j
return train_input_, train_output_, test_input_, test_output_