def normalize(self, gropuname1, groupname2): # ## normalize y ## # with h5py.File(self.OUTPATH, mode='r+') as f: for atom in self.MAINCHAIN: # load train_y = da.from_array( f[f'/{atom}/{gropuname1}/{self.RESPONSE_NAME}'], chunks=("auto", 3)) val_y = da.from_array( f[f'/{atom}/{groupname2}/{self.RESPONSE_NAME}'], chunks=("auto", 3)) total_y = da.concatenate([train_y, val_y], axis=0) y_mean = da.mean(total_y.reshape(-1), axis=0).compute() y_std = da.std(total_y.reshape(-1), axis=0).compute() # normalize train_y = da.divide(da.subtract(train_y, y_mean), y_std) val_y = da.divide(da.subtract(val_y, y_mean), y_std) # save da.to_hdf5(self.OUTPATH, f'/{atom}/{gropuname1}/{self.RESPONSE_NAME}', train_y) da.to_hdf5(self.OUTPATH, f'/{atom}/{groupname2}/{self.RESPONSE_NAME}', val_y) f.create_dataset(name=f'/{atom}/normalization', data=np.array([y_mean, y_std])) print(f'[{atom}]\tmean: {y_mean:.3f}\tstd: {y_std:.3f}')
def _preprocess(self, coords): # da.array function adjacent_coords = da.tile(coords, (1, self.N_ATOMS, 1)).reshape( coords.shape[0], self.N_ATOMS, self.N_ATOMS, 3) adjacent_coords = adjacent_coords.rechunk(chunks=('auto', -1, -1, -1)) descriptors = da.subtract(adjacent_coords, adjacent_coords.transpose(0, 2, 1, 3)) return descriptors
def weighting(spec_dist, temp_dist, comb_dist, similar_pixels_filtered): # Assign max weight (1) when the temporal or spectral distance is zero zero_spec_dist = da.where(spec_dist[:,mid_idx][:,None] == 1, 1, 0) zero_temp_dist = da.where(temp_dist[:,mid_idx][:,None] == 1, 1, 0) zero_dist_mid = da.where((zero_spec_dist == 1), zero_spec_dist, zero_temp_dist) shape = da.subtract(spec_dist.shape,(0,1)) zero_dist = da.zeros(shape, chunks=(spec_dist.shape[0],shape[1])) zero_dist = da.insert(zero_dist, [mid_idx], zero_dist_mid, axis=1) weights = da.where((da.sum(zero_dist,1)[:,None] == 1), zero_dist, comb_dist) # Calculate weights only for the filtered spectrally similar pixels weights_filt = weights*similar_pixels_filtered # Normalize weights norm_weights = da.rechunk(weights_filt/(da.sum(weights_filt,1)[:,None]), chunks = spec_dist.chunksize) print ("Done weighting!", norm_weights) return norm_weights