def normalize(self, gropuname1, groupname2):
# ## normalize y ## #
with h5py.File(self.OUTPATH, mode='r+') as f:
for atom in self.MAINCHAIN:
# load
train_y = da.from_array(
f[f'/{atom}/{gropuname1}/{self.RESPONSE_NAME}'],
chunks=("auto", 3))
val_y = da.from_array(
f[f'/{atom}/{groupname2}/{self.RESPONSE_NAME}'],
chunks=("auto", 3))
total_y = da.concatenate([train_y, val_y], axis=0)
y_mean = da.mean(total_y.reshape(-1), axis=0).compute()
y_std = da.std(total_y.reshape(-1), axis=0).compute()
# normalize
train_y = da.divide(da.subtract(train_y, y_mean), y_std)
val_y = da.divide(da.subtract(val_y, y_mean), y_std)
# save
da.to_hdf5(self.OUTPATH,
f'/{atom}/{gropuname1}/{self.RESPONSE_NAME}',
train_y)
da.to_hdf5(self.OUTPATH,
f'/{atom}/{groupname2}/{self.RESPONSE_NAME}', val_y)
f.create_dataset(name=f'/{atom}/normalization',
data=np.array([y_mean, y_std]))
print(f'[{atom}]\tmean: {y_mean:.3f}\tstd: {y_std:.3f}')
def _preprocess(self, coords): # da.array function
adjacent_coords = da.tile(coords, (1, self.N_ATOMS, 1)).reshape(
coords.shape[0], self.N_ATOMS, self.N_ATOMS, 3)
adjacent_coords = adjacent_coords.rechunk(chunks=('auto', -1, -1, -1))
descriptors = da.subtract(adjacent_coords,
adjacent_coords.transpose(0, 2, 1, 3))
return descriptors
def weighting(spec_dist, temp_dist, comb_dist, similar_pixels_filtered):
# Assign max weight (1) when the temporal or spectral distance is zero
zero_spec_dist = da.where(spec_dist[:,mid_idx][:,None] == 1, 1, 0)
zero_temp_dist = da.where(temp_dist[:,mid_idx][:,None] == 1, 1, 0)
zero_dist_mid = da.where((zero_spec_dist == 1),
zero_spec_dist, zero_temp_dist)
shape = da.subtract(spec_dist.shape,(0,1))
zero_dist = da.zeros(shape, chunks=(spec_dist.shape[0],shape[1]))
zero_dist = da.insert(zero_dist, [mid_idx], zero_dist_mid, axis=1)
weights = da.where((da.sum(zero_dist,1)[:,None] == 1), zero_dist, comb_dist)
# Calculate weights only for the filtered spectrally similar pixels
weights_filt = weights*similar_pixels_filtered
# Normalize weights
norm_weights = da.rechunk(weights_filt/(da.sum(weights_filt,1)[:,None]),
chunks = spec_dist.chunksize)
print ("Done weighting!", norm_weights)
return norm_weights