def test_preprocess_power(self):
main_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/deform/power.yml'))
shutil.rmtree(main_setting.data.preprocessed_root, ignore_errors=True)
preprocessor = pre.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
data_directory = main_setting.data.preprocessed_root \
/ 'train/tet2_3_modulusx0.9000'
preprocessed_x_grad = sp.load_npz(
data_directory / 'x_grad.npz')
reference_x_grad = sp.load_npz(
'tests/data/deform/interim/train/tet2_3_modulusx0.9000'
'/x_grad.npz').toarray()
with open(
main_setting.data.preprocessed_root / 'preprocessors.pkl',
'rb') as f:
preprocess_converter = pickle.load(f)['x_grad'][
'preprocess_converter']
std = preprocess_converter.converter.std_
np.testing.assert_almost_equal(
preprocessed_x_grad.toarray() * std**.5, reference_x_grad)
np.testing.assert_almost_equal(
preprocess_converter.converter.inverse_transform(
preprocessed_x_grad).toarray(), reference_x_grad)
def test_postprocessor(self):
data_setting = setting.DataSetting(
interim=Path('tests/data/prepost/interim'),
preprocessed=Path('tests/data/prepost/preprocessed'),
pad=False
)
preprocess_setting = setting.PreprocessSetting(
{
'identity': 'identity', 'std_scale': 'std_scale',
'standardize': 'standardize'}
)
main_setting = setting.MainSetting(
preprocess=preprocess_setting.preprocess, data=data_setting,
replace_preprocessed=False)
# Clean up data
shutil.rmtree(data_setting.interim_root, ignore_errors=True)
shutil.rmtree(data_setting.preprocessed_root, ignore_errors=True)
data_setting.preprocessed_root.mkdir(parents=True)
# Create data
interim_paths = [
data_setting.interim_root / 'a',
data_setting.interim_root / 'b']
for i, interim_path in enumerate(interim_paths):
interim_path.mkdir(parents=True)
n_element = np.random.randint(1e4)
identity = np.random.randint(2, size=(n_element, 1))
std_scale = np.random.rand(n_element, 3) * 5 * i
standardize = np.random.randn(n_element, 5) * 2 * i \
+ i * np.array([[.1, .2, .3, .4, .5]])
np.save(interim_path / 'identity.npy', identity)
np.save(interim_path / 'std_scale.npy', std_scale)
np.save(interim_path / 'standardize.npy', standardize)
(interim_path / 'converted').touch()
# Preprocess data
preprocessor = pre.Preprocessor(main_setting)
preprocessor.preprocess_interim_data()
postprocessor = pre.Converter(
data_setting.preprocessed_root / 'preprocessors.pkl')
preprocessed_paths = [
data_setting.preprocessed_root / 'a',
data_setting.preprocessed_root / 'b']
for interim_path, preprocessed_path in zip(
interim_paths, preprocessed_paths):
dict_data_x = {
'identity': np.load(preprocessed_path / 'identity.npy'),
'std_scale': np.load(preprocessed_path / 'std_scale.npy')}
dict_data_y = {
'standardize': np.load(preprocessed_path / 'standardize.npy')}
inv_dict_data_x, inv_dict_data_y, _ = postprocessor.postprocess(
dict_data_x, dict_data_y)
for k, v in inv_dict_data_x.items():
interim_data = np.load(interim_path / (k + '.npy'))
np.testing.assert_almost_equal(interim_data, v, decimal=5)
for k, v in inv_dict_data_y.items():
interim_data = np.load(interim_path / (k + '.npy'))
np.testing.assert_almost_equal(interim_data, v, decimal=5)
def preprocess_deform_timeseries():
main_setting = setting.MainSetting.read_settings_yaml(
pathlib.Path('tests/data/deform_timeseries/data.yml'))
preprocessor = prepost.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
return
def preprocess_deform():
main_setting = setting.MainSetting.read_settings_yaml(
pathlib.Path('tests/data/deform/data.yml'))
raw_converter = prepost.RawConverter(
main_setting,
recursive=True,
force_renew=True,
conversion_function=conversion_function)
raw_converter.convert()
preprocessor = prepost.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
interim_path = pathlib.Path(
'tests/data/deform/interim/train/tet2_3_modulusx0.9000')
preprocessed_path = pathlib.Path(
'tests/data/deform/preprocessed/train/tet2_3_modulusx0.9000')
interim_x_grad = sp.load_npz(interim_path / 'x_grad.npz')
preprocessed_x_grad = sp.load_npz(preprocessed_path / 'x_grad.npz')
scale_x_grad = preprocessed_x_grad.data / interim_x_grad.data
dict_reference_scale = calculate_scale_isoam()
np.testing.assert_almost_equal(np.var(scale_x_grad), 0.)
np.testing.assert_almost_equal(np.mean(scale_x_grad),
dict_reference_scale['x_grad'])
interim_y_grad = sp.load_npz(interim_path / 'y_grad.npz')
preprocessed_y_grad = sp.load_npz(preprocessed_path / 'y_grad.npz')
scale_y_grad = preprocessed_y_grad.data / interim_y_grad.data
np.testing.assert_almost_equal(np.mean(scale_y_grad),
np.mean(scale_x_grad))
return
def test_preprocess_same_as(self):
main_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/ode/data.yml'))
shutil.rmtree(main_setting.data.preprocessed_root, ignore_errors=True)
preprocessor = pre.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
data_directory = main_setting.data.preprocessed_root / 'train/0'
y0 = np.load(data_directory / 'y0.npy')
y0_initial = np.load(data_directory / 'y0_initial.npy')
np.testing.assert_almost_equal(y0[0], y0_initial[0])
np.testing.assert_almost_equal(y0_initial - y0_initial[0], 0.)
def test_preprocess_interim_list(self):
main_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/list/data.yml'))
shutil.rmtree(main_setting.data.preprocessed_root, ignore_errors=True)
preprocessor = pre.Preprocessor(main_setting)
preprocessor.preprocess_interim_data()
self.assertTrue(Path(
'tests/data/list/preprocessed/data/tet2_3_modulusx0.9500'
).exists())
self.assertTrue(Path(
'tests/data/list/preprocessed/data/tet2_4_modulusx0.9000'
).exists())
def preprocess_rotation():
main_setting = setting.MainSetting.read_settings_yaml(
pathlib.Path('tests/data/rotation/data.yml'))
raw_converter = prepost.RawConverter(
main_setting,
recursive=True,
force_renew=True,
conversion_function=rotation_conversion_function)
raw_converter.convert()
preprocessor = prepost.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
return
def preprocess_heat_time_series():
main_setting = setting.MainSetting.read_settings_yaml(
pathlib.Path('tests/data/heat_time_series/data.yml'))
raw_converter = prepost.RawConverter(
main_setting,
recursive=True,
force_renew=True,
conversion_function=conversion_function_heat_time_series,
write_ucd=False)
raw_converter.convert()
preprocessor = prepost.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
return
def test_preprocess_timeseries_data(self):
main_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/csv_timeseries/lstm.yml'))
shutil.rmtree(main_setting.data.preprocessed_root, ignore_errors=True)
p = pre.Preprocessor(main_setting)
p.preprocess_interim_data()
c = pre.Converter(
main_setting.data.preprocessed_root / 'preprocessors.pkl')
original_dict_x = {
'a': np.load(
main_setting.data.interim_root / 'train/0/a.npy')}
preprocessed_dict_x = c.preprocess(original_dict_x)
postprocessed_dict_x, _, _ = c.postprocess(preprocessed_dict_x, {})
np.testing.assert_almost_equal(
preprocessed_dict_x['a'],
np.load(
main_setting.data.preprocessed_root
/ 'train/0/a.npy'))
np.testing.assert_almost_equal(
original_dict_x['a'], postprocessed_dict_x['a'])
def test_preprocess_deform(self):
main_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/deform/data.yml'))
main_setting.data.interim = [Path(
'tests/data/deform/test_prepost/interim')]
main_setting.data.preprocessed = [Path(
'tests/data/deform/test_prepost/preprocessed')]
shutil.rmtree(main_setting.data.interim_root, ignore_errors=True)
shutil.rmtree(main_setting.data.preprocessed_root, ignore_errors=True)
raw_converter = pre.RawConverter(
main_setting, conversion_function=conversion_function)
raw_converter.convert()
p = pre.Preprocessor(main_setting)
p.preprocess_interim_data()
interim_strain = np.load(
'tests/data/deform/test_prepost/interim/train/'
'tet2_3_modulusx1.0000/elemental_strain.npy')
preprocessed_strain = np.load(
'tests/data/deform/test_prepost/preprocessed/train/'
'tet2_3_modulusx1.0000/elemental_strain.npy')
ratio_strain = interim_strain / preprocessed_strain
np.testing.assert_almost_equal(
ratio_strain - np.mean(ratio_strain), 0.)
interim_y_grad = sp.load_npz(
'tests/data/deform/test_prepost/interim/train/'
'tet2_3_modulusx1.0000/y_grad.npz')
preprocessed_y_grad = sp.load_npz(
'tests/data/deform/test_prepost/preprocessed/train/'
'tet2_3_modulusx1.0000/y_grad.npz')
ratio_y_grad = interim_y_grad.data \
/ preprocessed_y_grad.data
np.testing.assert_almost_equal(np.var(ratio_y_grad), 0.)
def test_preprocessor(self):
data_setting = setting.DataSetting(
interim=Path('tests/data/prepost/interim'),
preprocessed=Path('tests/data/prepost/preprocessed'),
pad=False
)
preprocess_setting = setting.PreprocessSetting(
{
'identity': 'identity', 'std_scale': 'std_scale',
'standardize': 'standardize'}
)
main_setting = setting.MainSetting(
preprocess=preprocess_setting.preprocess, data=data_setting,
replace_preprocessed=False)
main_setting.preprocess[ # pylint: disable=E1136
'std_scale']['componentwise'] = True # pylint: disable=E1136
main_setting.preprocess[ # pylint: disable=E1136
'standardize']['componentwise'] = True # pylint: disable=E1136
# Clean up data
shutil.rmtree(data_setting.interim_root, ignore_errors=True)
shutil.rmtree(data_setting.preprocessed_root, ignore_errors=True)
data_setting.preprocessed_root.mkdir(parents=True)
# Create data
interim_paths = [
data_setting.interim_root / 'a',
data_setting.interim_root / 'b']
for i, interim_path in enumerate(interim_paths):
interim_path.mkdir(parents=True)
n_element = int(1e5)
identity = np.random.randint(2, size=(n_element, 1))
std_scale = np.random.rand(n_element, 3) * 5 * i
standardize = np.random.randn(n_element, 5) * 2 * i \
+ i * np.array([[.1, .2, .3, .4, .5]])
np.save(interim_path / 'identity.npy', identity)
np.save(interim_path / 'std_scale.npy', std_scale)
np.save(interim_path / 'standardize.npy', standardize)
(interim_path / 'converted').touch()
# Preprocess data
preprocessor = pre.Preprocessor(main_setting)
preprocessor.preprocess_interim_data()
# Test preprocessed data is as desired
epsilon = 1e-5
preprocessed_paths = [
data_setting.preprocessed_root / 'a',
data_setting.preprocessed_root / 'b']
int_identity = np.concatenate([
np.load(p / 'identity.npy') for p in interim_paths])
pre_identity = np.concatenate([
np.load(p / 'identity.npy') for p in preprocessed_paths])
np.testing.assert_almost_equal(
int_identity, pre_identity, decimal=3)
int_std_scale = np.concatenate([
np.load(p / 'std_scale.npy') for p in interim_paths])
pre_std_scale = np.concatenate([
np.load(p / 'std_scale.npy') for p in preprocessed_paths])
np.testing.assert_almost_equal(
int_std_scale / (np.std(int_std_scale, axis=0) + epsilon),
pre_std_scale, decimal=3)
np.testing.assert_almost_equal(
np.std(pre_std_scale), 1. + epsilon, decimal=3)
int_standardize = np.concatenate([
np.load(p / 'standardize.npy') for p in interim_paths])
pre_standardize = np.concatenate([
np.load(p / 'standardize.npy') for p in preprocessed_paths])
np.testing.assert_almost_equal(
(int_standardize - np.mean(int_standardize, axis=0))
/ (np.std(int_standardize, axis=0) + epsilon),
pre_standardize, decimal=3)
np.testing.assert_almost_equal(
np.std(pre_standardize, axis=0), 1. + epsilon, decimal=3)
np.testing.assert_almost_equal(
np.mean(pre_standardize, axis=0), np.zeros(5), decimal=3)
def test_grad_grad_vs_laplacian(self):
preprocess_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/precalculated/convert.yml'))
shutil.rmtree(preprocess_setting.data.interim, ignore_errors=True)
shutil.rmtree(preprocess_setting.data.preprocessed, ignore_errors=True)
raw_converter = pre.RawConverter(
preprocess_setting,
conversion_function=convert.HeatConversionFuncionCreator(
create_elemental=False),
filter_function=convert.filter_function_heat,
force_renew=True, recursive=True, to_first_order=True,
write_ucd=False, read_npy=False, read_res=True)
raw_converter.convert()
preprocessor = pre.Preprocessor(
preprocess_setting, force_renew=True)
preprocessor.preprocess_interim_data()
# Confirm results does not change under rigid body transformation
grad_grad_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/precalculated/grad_grad.yml'))
grad_grad_tr = trainer.Trainer(grad_grad_setting)
if grad_grad_tr.setting.trainer.output_directory.exists():
shutil.rmtree(grad_grad_tr.setting.trainer.output_directory)
grad_grad_tr.train()
grad_grad_ir = inferer.Inferer(grad_grad_setting)
inference_outpout_directory = \
grad_grad_setting.trainer.output_directory / 'inferred'
if inference_outpout_directory.exists():
shutil.rmtree(inference_outpout_directory)
results = grad_grad_ir.infer(
model=Path('tests/data/precalculated/models/grad_grad'),
preprocessed_data_directory=[
Path(
'tests/data/precalculated/preprocessed/cube/clscale1.0/'
'original'),
Path(
'tests/data/precalculated/preprocessed/cube/clscale1.0/'
'rotated')],
converter_parameters_pkl=Path(
'tests/data/precalculated/preprocessed/preprocessors.pkl'),
output_directory=inference_outpout_directory,
overwrite=True)
np.testing.assert_almost_equal(
results[0]['dict_y']['nodal_t_100'],
results[1]['dict_y']['nodal_t_100'])
laplacian_setting = setting.MainSetting.read_settings_yaml(
Path('tests/data/precalculated/laplacian.yml'))
laplacian_tr = trainer.Trainer(laplacian_setting)
if laplacian_tr.setting.trainer.output_directory.exists():
shutil.rmtree(laplacian_tr.setting.trainer.output_directory)
laplacian_tr.train()
laplacian_ir = inferer.Inferer(laplacian_setting)
inference_outpout_directory = \
laplacian_setting.trainer.output_directory / 'inferred'
if inference_outpout_directory.exists():
shutil.rmtree(inference_outpout_directory)
results = laplacian_ir.infer(
model=Path('tests/data/precalculated/models/laplacian'),
preprocessed_data_directory=[
Path(
'tests/data/precalculated/preprocessed/cube/clscale1.0/'
'original'),
Path(
'tests/data/precalculated/preprocessed/cube/clscale1.0/'
'rotated')],
converter_parameters_pkl=Path(
'tests/data/precalculated/preprocessed/preprocessors.pkl'),
output_directory=inference_outpout_directory,
overwrite=True)
np.testing.assert_almost_equal(
results[0]['dict_y']['nodal_t_100'],
results[1]['dict_y']['nodal_t_100'])
def generate_ode():
time_range = (10., 50.)
delta_t = .1
def f0(ts, xs):
ys = np.zeros(list(xs.shape[:2]) + [1])
ys[0] = np.random.rand(*list(ys.shape)[1:])
for i in range(1, len(ts)):
ys[i, :, 0] = ys[i - 1, :, 0] + delta_t * (-.1 * ys[i - 1, :, 0])
return ys
def f1(ts, xs):
ys = np.zeros(list(xs.shape[:2]) + [1])
ys[0] = np.random.rand(*list(ys.shape)[1:])
for i in range(1, len(ts)):
ys[i, :, 0] = ys[i - 1, :, 0] + delta_t * xs[i, :, 0] * .1
return ys
def f2(ts, xs):
ys = np.zeros(list(xs.shape[:2]) + [1])
ys[0] = np.random.rand(*list(ys.shape)[1:])
for i in range(1, len(ts)):
ys[i, :,
0] = ys[i - 1, :,
0] + delta_t * (.01 * xs[i, :, 1] - .01 * xs[i, :, 0] *
xs[i, :, 3] - .01 * ys[i - 1, :, 0])
return ys
def f3(ts, xs):
ys = np.zeros(list(xs.shape[:2]) + [2])
ys[0] = np.random.rand(*list(ys.shape)[1:]) * 2 - 1
for i in range(1, len(ts)):
ys[i, :, 0] = ys[i - 1, :, 0] + delta_t * (
-.05 * ys[i - 1, :, 1] + .01 *
(1 - ys[i - 1, :, 1]**2) * ys[i - 1, :, 0])
ys[i, :, 1] = ys[i - 1, :, 1] + delta_t * ys[i - 1, :, 0]
return ys
def generate_ode(root_dir, n_data):
if root_dir.exists():
shutil.rmtree(root_dir)
for i in range(n_data):
n_element = np.random.randint(3, 10)
t_max = np.random.rand() * (time_range[1] -
time_range[0]) + time_range[0]
ts = np.arange(0., t_max, delta_t)
x0 = np.random.rand() * np.sin(
2 * np.pi * (np.random.rand() / 10. * ts + np.random.rand()))
x1 = np.random.rand() * np.sin(
2 * np.pi * (np.random.rand() / 20. * ts + np.random.rand()))
x2 = np.random.rand() * (
1 - np.exp(-ts / 5. * np.random.rand())) + np.random.rand()
x3 = np.exp(-ts / 10. * np.random.rand()) + np.random.rand()
_xs = np.stack([x0, x1, x2, x3], axis=1)[:, None, :]
xs = np.concatenate(
[_xs * a for a in np.linspace(1., 2., n_element)], axis=1)
y0 = f0(ts, xs)
y1 = f1(ts, xs)
y2 = f2(ts, xs)
y3 = f3(ts, xs)
stacked_ts = np.concatenate([ts[:, None, None]] * n_element,
axis=1)
output_directory = root_dir / f"{i}"
output_directory.mkdir(parents=True)
np.save(output_directory / 't.npy', stacked_ts.astype(np.float32))
np.save(output_directory / 'x.npy', xs.astype(np.float32))
np.save(output_directory / 'y0.npy', y0.astype(np.float32))
np.save(output_directory / 'y1.npy', y1.astype(np.float32))
np.save(output_directory / 'y2.npy', y2.astype(np.float32))
np.save(output_directory / 'y3.npy', y3.astype(np.float32))
np.save(output_directory / 'y0_initial.npy',
(np.ones(y0.shape) * y0[0, :, :]).astype(np.float32))
np.save(output_directory / 'y1_initial.npy',
(np.ones(y1.shape) * y1[0, :, :]).astype(np.float32))
np.save(output_directory / 'y2_initial.npy',
(np.ones(y2.shape) * y2[0, :, :]).astype(np.float32))
np.save(output_directory / 'y3_initial.npy',
(np.ones(y3.shape) * y3[0, :, :]).astype(np.float32))
(output_directory / 'converted').touch()
if PLOT:
plt.plot(ts, x0, label='x0')
plt.plot(ts, x1, label='x1')
plt.plot(ts, x2, label='x2')
plt.plot(ts, x3, label='x3')
plt.plot(ts, y0[:, 0, 0], label='y0')
plt.plot(ts, y1[:, 0, 0], label='y1')
plt.plot(ts, y2[:, 0, 0], label='y2')
plt.plot(ts, y3[:, 0, 0], label='y3-0')
plt.plot(ts, y3[:, 0, 1], label='y3-1')
plt.legend()
plt.savefig(output_directory / 'plot.pdf')
plt.show()
return
generate_ode(pathlib.Path('tests/data/ode/interim/train'), 100)
generate_ode(pathlib.Path('tests/data/ode/interim/validation'), 10)
generate_ode(pathlib.Path('tests/data/ode/interim/test'), 10)
main_setting = setting.MainSetting.read_settings_yaml(
pathlib.Path('tests/data/ode/data.yml'))
preprocessor = prepost.Preprocessor(main_setting, force_renew=True)
preprocessor.preprocess_interim_data()
return
