def vis_curves(curves):
i = 0
for c, _ in curves:
print('computing...')
curve_batch = np.array(c)
grids = generator.predict(curve_batch)
intermediate_layer_outputs = list()
for layer in vis_layers:
intermediate = layer.predict(curve_batch)
print(intermediate.shape)
intermediate_layer_outputs.append(intermediate)
densities, _ = run_dft(grids, inner_loops=100)
for diffs, grid, diffs_dft, sample1, sample2, sample3 in zip(c, grids, densities, *intermediate_layer_outputs):
curve = np.cumsum(np.insert(diffs, 0, 0))
curve_dft = np.cumsum(np.insert(diffs_dft, 0, 0))
error = np.sum(np.abs(curve - curve_dft)) / len(curve)
errors.append(error)
area = np.sum(curve_dft) / len(curve_dft)
areas.append(area)
if see_grids:
'''
os.makedirs(f'figures/random_curves/{i}', exist_ok=True)
np.savetxt(f'figures/random_curves/{i}/grid.csv', grid, delimiter=',')
np.savetxt(f'figures/random_curves/{i}/curve.csv', curve, delimiter=',')
np.savetxt(f'figures/random_curves/{i}/curve_dft.csv', curve_dft, delimiter=',')
'''
i += 1
show_grid(grid, curve, curve_dft)
plt.show()
def make_dft_model():
inp = Input(shape=(GRID_SIZE, GRID_SIZE), batch_size=generator_batchsize, name='dft_input')
x = Lambda(lambda x: run_dft(x,
batch_size=generator_batchsize,
inner_loops=inner_loops)[0])(inp)
model = Model(inputs=inp, outputs=x, name='dft_model')
return model
def main(inner_loops):
base_dir = './data_generation'
density_files = glob.glob(
os.path.join(base_dir, 'results', 'density_*.csv'))
density_files.sort(reverse=False)
density_files = density_files[:]
real_densities_diffs = [
np.diff(np.genfromtxt(density_file, delimiter=','))
for density_file in density_files
]
shuffle(real_densities_diffs)
tile_size = GRID_SIZE // 20
lat_size = tile_size * tile_size
sample_grids = list()
effective_weights = list()
# 19 * 19 = 361 which is approximately 360
# aim for half cells are pores ~180 pores
# half
for pore_size in range(1, 19 // 2):
grid = np.zeros((GRID_SIZE, GRID_SIZE), dtype=np.float32)
for i in range(0, 19, 2 * pore_size):
for j in range(0, 19, 2 * pore_size):
grid[i:i + pore_size, j:j + pore_size] = 1
grid[i + pore_size:i + 2 * pore_size,
j + pore_size:j + 2 * pore_size] = 1
sample_grids.append(grid)
for width in range(1, 19 // 2):
grid = np.zeros((GRID_SIZE, GRID_SIZE), dtype=np.float32)
for i in range(0, 19, 2 * width):
grid[i:i + width] = 1
sample_grids.append(grid)
# 19 * 19 = 361 now we go for the full size pores
for i in range(15, 20):
grid = np.ones((GRID_SIZE, GRID_SIZE), dtype=np.float32)
sample_grids.append(grid)
mask = np.zeros((GRID_SIZE, GRID_SIZE), dtype=np.float32)
mask[:19, :19] = 1
sample_grids = np.array([g * mask for g in sample_grids])
effective_weights = np.array([np.sum(g) for g in sample_grids])
sample_curves_diffs, _ = run_dft(sample_grids, inner_loops=inner_loops)
sample_curves_diffs = sample_curves_diffs.numpy()
sample_curves = np.cumsum(sample_curves_diffs, axis=1)
if 's' in sys.argv[1:]:
for grid, curve_diffs in zip(sample_grids, sample_curves_diffs):
fig = plt.figure(figsize=(10, 4))
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
ax = plt.subplot(1, 2, 1)
ax.clear()
ax.set_title('Grid (Black = Solid, Whited = Pore)')
ax.set_yticks(np.linspace(0, grid.shape[0], 5))
ax.set_xticks(np.linspace(0, grid.shape[1], 5))
ax.pcolor(1 - grid, cmap='Greys', vmin=0.0, vmax=1.0)
ax.set_aspect('equal')
actual_curve = np.insert(np.cumsum(curve_diffs), 0, 0)
ax = plt.subplot(1, 2, 2)
ax.clear()
ax.set_title('Adsorption Curve')
ax.set_xticks(np.linspace(0, 1, 10))
ax.set_yticks(np.linspace(0, 1, 5))
ax.set_ylim(0, 1)
ax.plot(np.linspace(0, 1, N_ADSORP + 1), actual_curve, color='red')
ax.scatter(np.linspace(0, 1, N_ADSORP), curve_diffs)
plt.show()
generated_grids = list()
weighting_errs = list()
norm_weighting_errs = list()
generator_errs = list()
target_curve_areas = list()
for i, target_curve_diffs in enumerate(real_densities_diffs):
target_curve = np.insert(np.cumsum(target_curve_diffs), 0, 0)
target_curve_area = np.sum(target_curve) / len(target_curve)
target_curve_areas.append(target_curve_area)
weights = gen_curve_weights(target_curve[1:], sample_curves, 80)
weights /= effective_weights
weights /= np.sum(weights)
weighted_target_curve = weights.T @ sample_curves
weighted_target_curve = np.insert(weighted_target_curve, 0, 0)
err = np.sum(
np.abs(target_curve - weighted_target_curve)) / len(target_curve)
weighting_errs.append(err)
count_weights, norm_weights = round(weights, lat_size=lat_size)
count_weights = count_weights.astype(np.int32)
norm_weighted_target_curve = norm_weights.T @ sample_curves
norm_weighted_target_curve = np.insert(norm_weighted_target_curve, 0,
0)
err = np.sum(np.abs(target_curve -
norm_weighted_target_curve)) / len(target_curve)
norm_weighting_errs.append(err)
generated_grid = np.zeros((GRID_SIZE, GRID_SIZE), dtype=np.float32)
fill(generated_grid, count_weights, sample_grids, lat_size=lat_size)
generated_grids.append(generated_grid)
print(f'\r {i}/{len(real_densities_diffs)}', end='')
'''
generated_curve_diffs, _ = run_dft(np.array([generated_grid]), inner_loops=inner_loops)
generated_curve = np.insert(np.cumsum(generated_curve_diffs), 0, 0)
err = np.sum(np.abs(target_curve - generated_curve)) / len(target_curve)
generator_errs.append(err)
e = np.array(generator_errs)
print(f'\r {i}/{len(real_densities_diffs)} | Error: {e.mean():.5f} | Max: {e.max():.5f} | Min: {e.min():.5f} | Std: {e.std():.5f}', end='')
'''
if 'g' not in sys.argv[1:]:
continue
fig = plt.figure(figsize=(10, 4))
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
ax = plt.subplot(1, 2, 1)
ax.clear()
ax.set_title('Grid (Black = Solid, Whited = Pore)')
ax.set_yticks(np.linspace(0, generated_grid.shape[0], 5))
ax.set_xticks(np.linspace(0, generated_grid.shape[1], 5))
ax.pcolor(1 - generated_grid, cmap='Greys', vmin=0.0, vmax=1.0)
ax.set_aspect('equal')
ax = plt.subplot(1, 2, 2)
ax.clear()
ax.set_title('Adsorption Curve')
ax.set_xticks(np.linspace(0, 1, 10))
ax.set_yticks(np.linspace(0, 1, 5))
ax.set_ylim(0, 1)
ax.plot(np.linspace(0, 1, N_ADSORP + 1),
target_curve,
label='target curve')
# ax.plot(np.linspace(0, 1, N_ADSORP+1), weighted_target_curve, label='weighted curve')
# ax.plot(np.linspace(0, 1, N_ADSORP+1), norm_weighted_target_curve, label='norm weighted curve')
ax.plot(np.linspace(0, 1, N_ADSORP + 1),
generated_curve,
label='generated curve')
plt.legend()
plt.show()
print()
##############################################
dft_batch_size = 128
generated_grids = np.array(generated_grids)
generated_curve_diffs = list()
for i in range(0, len(generated_grids), dft_batch_size):
print(f'\r {i}/{len(generated_grids)}', end='')
gen_diffs, _ = run_dft(generated_grids[i:i + dft_batch_size],
inner_loops=100)
generated_curve_diffs.extend(gen_diffs)
print()
generated_curve_diffs = np.array(generated_curve_diffs)
generated_curves = np.cumsum(generated_curve_diffs, axis=1)
for i, (target_curve_diffs, generated_curve) in enumerate(
zip(real_densities_diffs, generated_curves)):
target_curve = np.insert(np.cumsum(target_curve_diffs), 0, 0)
generated_curve = np.insert(generated_curve, 0, 0)
err = np.sum(
np.abs(target_curve - generated_curve)) / len(target_curve)
generator_errs.append(err)
e = np.array(generator_errs)
print(
f'\r {i}/{len(real_densities_diffs)} | Error: {e.mean():.5f} | Max: {e.max():.5f} | Min: {e.min():.5f} | Std: {e.std():.5f}',
end='')
print()
##############################################
target_curve_areas = np.array(target_curve_areas)
for errs, name in zip(
[weighting_errs, norm_weighting_errs, generator_errs],
['weighting', 'normed', 'gen']):
errs = np.array(errs)
print(f'{name}: {errs.mean():.5f} | {errs.std():.5f}')
os.makedirs(f'figures/quick_inverse_error/', exist_ok=True)
area_err = np.concatenate(
(np.array(target_curve_areas)[:, None], np.array(errs)[:, None]),
axis=1)
np.savetxt(f'figures/quick_inverse_error/errors_{name}.csv',
area_err,
delimiter=',')
plt.hist(errs, bins=20)
plt.title(name)
plt.show()
plt.scatter(target_curve_areas, errs)
plt.title(f'{name} w.r.t area')
plt.show()
exit(0)
def visualize(see_grids, intermediate_layers):
generator = inverse_dft_model()
generator.load_weights(model_loc, by_name=True)
vis_layers = list()
for layer in generator.layers:
if 'conv2d' in layer.name or 'batch_normalization' == layer.name:
vis_layers.append(Model(generator.input, layer.output))
relative_humidity = np.arange(41) * STEP_SIZE
areas = list()
errors = list()
c = make_steps()[0][::]
square_grids = np.zeros((20, 20, 20), dtype=np.float32)
for i in range(1, 21):
square_grids[i-1, :i, :i] = 1
square_curves, _ = run_dft(square_grids, inner_loops=300)
c = np.concatenate((square_curves, c))
straight_line = np.zeros((1, 40), dtype=np.float32)
straight_line[0, 0:40] = 1/40
c = np.concatenate((straight_line, c))
grids = generator.predict(c)
densities, _ = run_dft(grids, inner_loops=300)
intermediate_layer_outputs = list()
for layer in vis_layers:
intermediate = layer.predict(c)
intermediate_layer_outputs.append(intermediate)
for i, (diffs, grid, diffs_dft, sample1, sample2, sample3) in enumerate(zip(c, grids, densities, *intermediate_layer_outputs)):
curve = np.cumsum(np.insert(diffs, 0, 0))
curve_dft = np.cumsum(np.insert(diffs_dft, 0, 0))
if see_grids:
'''
os.makedirs(f'figures/artificial/{i}', exist_ok=True)
np.savetxt(f'figures/artificial/{i}/grid.csv', grid, delimiter=',')
np.savetxt(f'figures/artificial/{i}/curve.csv', curve, delimiter=',')
np.savetxt(f'figures/artificial/{i}/curve_dft.csv', curve_dft, delimiter=',')
'''
# show_grid(grid, curve, curve_dft)
# plt.show()
'''
fig, ax = plt.subplots(8, 16, figsize=(16, 8))
for i in range(128):
ax[i//16, i % 16].matshow(sample1[:, :, i])
plt.show()
fig, ax = plt.subplots(8, 16, figsize=(16, 8))
for i in range(128):
ax[i//16, i % 16].matshow(sample2[:, :, i])
plt.show()
fig, ax = plt.subplots(8, 16, figsize=(16, 8))
for i in range(128):
ax[i//16, i % 16].matshow(sample3[:, :, i])
plt.show()
'''
def vis_curves(curves):
i = 0
for c, _ in curves:
print('computing...')
curve_batch = np.array(c)
grids = generator.predict(curve_batch)
intermediate_layer_outputs = list()
for layer in vis_layers:
intermediate = layer.predict(curve_batch)
print(intermediate.shape)
intermediate_layer_outputs.append(intermediate)
densities, _ = run_dft(grids, inner_loops=100)
for diffs, grid, diffs_dft, sample1, sample2, sample3 in zip(c, grids, densities, *intermediate_layer_outputs):
curve = np.cumsum(np.insert(diffs, 0, 0))
curve_dft = np.cumsum(np.insert(diffs_dft, 0, 0))
error = np.sum(np.abs(curve - curve_dft)) / len(curve)
errors.append(error)
area = np.sum(curve_dft) / len(curve_dft)
areas.append(area)
if see_grids:
'''
os.makedirs(f'figures/random_curves/{i}', exist_ok=True)
np.savetxt(f'figures/random_curves/{i}/grid.csv', grid, delimiter=',')
np.savetxt(f'figures/random_curves/{i}/curve.csv', curve, delimiter=',')
np.savetxt(f'figures/random_curves/{i}/curve_dft.csv', curve_dft, delimiter=',')
'''
i += 1
show_grid(grid, curve, curve_dft)
plt.show()
# fig, ax = plt.subplots(8, 16, figsize=(16, 8))
# for i in range(128):
# ax[i//16, i % 16].matshow(sample1[:, :, i])
# plt.show()
# fig, ax = plt.subplots(8, 16, figsize=(16, 8))
# for i in range(128):
# ax[i//16, i % 16].matshow(sample2[:, :, i])
# plt.show()
# fig, ax = plt.subplots(8, 16, figsize=(16, 8))
# for i in range(128):
# ax[i//16, i % 16].matshow(sample3[:, :, i])
# plt.show()
# curves = [next(generator_train_generator()) for _ in range(5)]
# vis_curves(curves)
# base_dir = './test_grids/step4'
base_dir = './data_generation'
density_files = glob.glob(os.path.join(base_dir, 'results', 'density_*.csv'))
density_files.sort(reverse=False)
density_files = density_files[:]
true_densities = [np.diff(np.genfromtxt(density_file, delimiter=',')) for density_file in density_files]
shuffle(true_densities)
# true_densities = batch(true_densities, generator_batchsize)
true_densities = batch(true_densities, 256)
vis_curves(zip(true_densities, true_densities))
print('Mean: ', np.array(errors).mean())
print('Std: ', np.array(errors).std())
plt.hist(errors, bins=15)
plt.title('Error Distribution')
plt.xlabel('Abs error')
# plt.xlim(0, 1)
plt.show()
plt.scatter(areas, errors)
print(np.array(areas).shape)
print(np.array(areas).dtype)
print(np.array(errors).shape)
print(np.array(errors).dtype)
os.makedirs(f'figures/generator_error/', exist_ok=True)
np.savetxt('figures/generator_error/errors.csv', np.concatenate((np.array(areas)[:, None], np.array(errors)[:, None]), axis=1), delimiter=',')
plt.title('Error w.r.t. area under curve')
plt.xlabel('Area under DFT curve')
plt.ylabel('Abs error')
# plt.xlim(0, 1)
# plt.ylim(0, 1)
plt.show()