def store_train_h5(self, path): if os.path.exists(path): os.remove(path) dpack = pyt.datapacker(path) for j, (i, X, F, E, S, P) in enumerate( zip(self.kid, self.xyz, self.frc, self.Eqm, self.spc, self.prt)): xyz = X[i] frc = F[i] eng = E[i] spc = S nme = P # Prepare and store the test data set if xyz.size != 0: dpack.store_data(nme + '/mol' + str(j), coordinates=xyz, forces=frc, energies=eng, species=spc) # Cleanup the disk dpack.cleanup()
def generate_h5(self, path): # open an HDF5 for compressed storage. # Note that if the path exists, it will open whatever is there. dpack = pyt.datapacker(path) d = self.ldtdir + self.datdir + '/data/' files = [f for f in os.listdir(d) if ".dat" in f] files.sort() Nf = len(files) Nd = 0 for n, f in enumerate(files): #print(d + f) L = file_len(d + f) if L >= 4: #print(d + f) data = hdt.readncdatall(d + f) if 'energies' in data: Ne = data['energies'].size Nd += Ne f = f.rsplit("-", 1) fn = f[0] + "/mol" + f[1].split(".")[0] dpack.store_data(fn, **data) dpack.cleanup()
def store_data(self, filename): if os.path.exists(filename): os.remove(filename) dpack = ant.datapacker(filename) for k in self.tdata.keys(): dpack.store_data(k, **(self.tdata[k])) dpack.cleanup()
def __init__(self, hdf5files, saef, output, storecac, storetest, Naev): self.xyz = [] self.frc = [] self.Eqm = [] self.spc = [] self.idx = [] self.gid = [] self.prt = [] self.Naev = Naev self.kid = [] # list to track data kept self.nt = [] # total conformers self.nc = [] # total kept self.of = open(output, 'w') self.tf = 0 for f in hdf5files: # Construct the data loader class adl = pyt.anidataloader(f) print('Loading file:', f) # Declare test cache if os.path.exists(storetest): os.remove(storetest) dpack = pyt.datapacker(storetest) for i, data in enumerate(adl): xyz = data['coordinates'] frc = data['forces'] eng = data['energies'] spc = data['species'] nme = data['path'] # Toss out high forces Mv = np.max(np.linalg.norm(frc, axis=2), axis=1) index = np.where(Mv > 1.75)[0] indexk = np.where(Mv <= 1.75)[0] # CLear forces xyz = xyz[indexk] frc = frc[indexk] eng = eng[indexk] idx = np.random.uniform(0.0, 1.0, eng.size) tr_idx = np.asarray(np.where(idx < 0.99))[0] te_idx = np.asarray(np.where(idx >= 0.99))[0] #print(tr_idx) if tr_idx.size > 0: self.prt.append(nme) self.xyz.append( np.ndarray.astype(xyz[tr_idx], dtype=np.float32)) self.frc.append( np.ndarray.astype(frc[tr_idx], dtype=np.float32)) self.Eqm.append( np.ndarray.astype(eng[tr_idx], dtype=np.float64)) self.spc.append(spc) Nd = eng[tr_idx].size #print(Nd) self.idx.append(np.arange(Nd)) self.kid.append(np.array([], dtype=np.int)) self.gid.append(np.array([], dtype=np.int)) self.tf = self.tf + Nd self.nt.append(Nd) self.nc.append(0) # Prepare and store the test data set if xyz[te_idx].size != 0: #t_xyz = xyz[te_idx].reshape(te_idx.size, xyz[te_idx].shape[1] * xyz[te_idx].shape[2]) dpack.store_data(nme + '/mol' + str(i), coordinates=xyz[te_idx], forces=frc[te_idx], energies=np.array(eng[te_idx]), species=spc) # Clean up adl.cleanup() # Clean up dpack.cleanup() self.nt = np.array(self.nt) self.nc = np.array(self.nc) self.ts = 0 self.vs = 0 self.Nbad = self.tf self.saef = saef self.storecac = storecac
# ds.append(np.array(tidx).size) # tmol += 1 #print('total data:',tdat) #ddif = tdat - mtdat #ds = np.array(ds) #cnt = np.ones(ddif,dtype=np.int32) #rmtot = np.zeros(tmol,dtype=np.int32) #for i in range(tmol): # rmtot[i] = cnt[] #print(rmtot) dp = pyt.datapacker(h5dir) #print('Diff data:',ddif,'tmol:',tmol) tmol = 0 gcount = 0 bcount = 0 for i, f in enumerate(files): X = [] E = [] tmol += 1 for e in ends: data = hdn.readncdat(dtdir + f + e + '.dat') X.append(np.array(data[0], dtype=np.float32)) E.append(np.array(data[2], dtype=np.float64)) S = data[1] X = np.concatenate(X)
if os.path.exists(store_dir + str(i) + '/testset/testset.h5'): os.remove(store_dir + str(i) + '/testset/testset.h5') if not os.path.exists(store_dir + str(i) + '/testset'): os.mkdir(store_dir + str(i) + '/testset') cachet = [ cg('_train', saef, store_dir + str(r) + '/', forcet, chargt, False) for r in range(N) ] cachev = [ cg('_valid', saef, store_dir + str(r) + '/', forcet, chargt, False) for r in range(N) ] testh5 = [ pyt.datapacker(store_dir + str(r) + '/testset/testset.h5') for r in range(N) ] Nd = np.zeros(N, dtype=np.int32) Nbf = 0 for f, fn in enumerate(h5files): print('Processing file(' + str(f + 1) + ' of ' + str(len(h5files)) + '):', fn) adl = pyt.anidataloader(fn) To = adl.size() Ndc = 0 Fmt = [] Emt = [] for c, data in enumerate(adl):
''' hdf5file = '/home/jujuman/Research/ANI-DATASET/ani_data_c01test.h5' storecac = '/home/jujuman/Research/GDB-11-wB97X-6-31gd/cache01_2/' saef = "/home/jujuman/Research/GDB-11-wB97X-6-31gd/sae_6-31gd.dat" path = "/home/jujuman/Research/GDB-11-wB97X-6-31gd/cache01_2/testset/c01-testset.h5" ''' # Construct the data loader class adl = pya.anidataloader(hdf5file) # Declare data cache cachet = cg('_train', saef, storecac) cachev = cg('_valid', saef, storecac) # Declare test cache dpack = pyt.datapacker(path) # Load morse parameters popt = np.load('mp_ani_params_test.npz')['param'] # Loop over data in set for data in adl.getnextdata(): loc = data['parent'] + "/" + data['child'] print(loc) xyz = data['coordinates'] eng = data['energies'] spc = data['species'] # Compute Morse Potential sdat = [
def build_strided_training_cache(self, Nblocks, Nvalid, Ntest, build_test=True, build_valid=False, forces=True, grad=False, Fkey='forces', forces_unit=1.0, Ekey='energies', energy_unit=1.0, Eax0sum=False, rmhighe=True): if not os.path.isfile(self.netdict['saefile']): self.sae_linear_fitting(Ekey=Ekey, energy_unit=energy_unit, Eax0sum=Eax0sum) h5d = self.h5dir store_dir = self.train_root + "cache-data-" N = self.Nn Ntrain = Nblocks - Nvalid - Ntest if Nblocks % N != 0: raise ValueError( 'Error: number of networks must evenly divide number of blocks.' ) Nstride = Nblocks / N for i in range(N): if not os.path.exists(store_dir + str(i)): os.mkdir(store_dir + str(i)) if build_test: if os.path.exists(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5'): os.remove(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5') if not os.path.exists(store_dir + str(i) + '/../testset'): os.mkdir(store_dir + str(i) + '/../testset') cachet = [ cg('_train', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] cachev = [ cg('_valid', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] if build_test: testh5 = [ pyt.datapacker(store_dir + str(r) + '/../testset/testset' + str(r) + '.h5') for r in range(N) ] if build_valid: valdh5 = [ pyt.datapacker(store_dir + str(r) + '/../testset/valdset' + str(r) + '.h5') for r in range(N) ] if rmhighe: dE = [] for f in self.h5file: adl = pyt.anidataloader(h5d + f) for data in adl: S = data['species'] E = data['energies'] X = data['coordinates'] Esae = hdt.compute_sae(self.netdict['saefile'], S) dE.append((E - Esae) / np.sqrt(len(S))) dE = np.concatenate(dE) cidx = np.where(np.abs(dE) < 15.0) std = np.abs(dE[cidx]).std() men = np.mean(dE[cidx]) print(men, std, men + std) idx = np.intersect1d( np.where(dE >= -np.abs(15 * std + men))[0], np.where(dE <= np.abs(11 * std + men))[0]) cnt = idx.size print('DATADIST: ', dE.size, cnt, (dE.size - cnt), 100.0 * ((dE.size - cnt) / dE.size)) E = [] data_count = np.zeros((N, 3), dtype=np.int32) for f in self.h5file: print('Reading data file:', h5d + f) adl = pyt.anidataloader(h5d + f) for data in adl: #print(data['path'],data['energies'].size) S = data['species'] if data[Ekey].size > 0 and (set(S).issubset( self.netdict['atomtyp'])): X = np.array(data['coordinates'], order='C', dtype=np.float32) #print(np.array(data[Ekey].shape),np.sum(np.array(data[Ekey], order='C', dtype=np.float64),axis=1).shape,data[Fkey].shape) if Eax0sum: E = energy_unit * np.sum(np.array( data[Ekey], order='C', dtype=np.float64), axis=1) else: E = energy_unit * np.array( data[Ekey], order='C', dtype=np.float64) if forces and not grad: F = forces_unit * np.array( data[Fkey], order='C', dtype=np.float32) elif forces and grad: F = -forces_unit * np.array( data[Fkey], order='C', dtype=np.float32) else: F = 0.0 * X if rmhighe: Esae = hdt.compute_sae(self.netdict['saefile'], S) ind_dE = (E - Esae) / np.sqrt(len(S)) hidx = np.union1d( np.where(ind_dE < -(15.0 * std + men))[0], np.where(ind_dE > (11.0 * std + men))[0]) lidx = np.intersect1d( np.where(ind_dE >= -(15.0 * std + men))[0], np.where(ind_dE <= (11.0 * std + men))[0]) if hidx.size > 0: print( ' -(' + f + ':' + data['path'] + ')High energies detected:\n ', (E[hidx] - Esae) / np.sqrt(len(S))) X = X[lidx] E = E[lidx] F = F[lidx] # Build random split index ridx = np.random.randint(0, Nblocks, size=E.size) Didx = [ np.argsort(ridx)[np.where(ridx == i)] for i in range(Nblocks) ] # Build training cache for nid, cache in enumerate(cachet): set_idx = np.concatenate([ Didx[((bid + nid * int(Nstride)) % Nblocks)] for bid in range(Ntrain) ]) if set_idx.size != 0: data_count[nid, 0] += set_idx.size cache.insertdata(X[set_idx], F[set_idx], E[set_idx], list(S)) # for nid,cache in enumerate(cachev): # set_idx = np.concatenate([Didx[((1+bid+nid*int(Nstride)) % Nblocks)] for bid in range(Ntrain)]) # if set_idx.size != 0: # data_count[nid,0]+=set_idx.size # cache.insertdata(X[set_idx], F[set_idx], E[set_idx], list(S)) for nid, cache in enumerate(cachev): set_idx = np.concatenate([ Didx[(Ntrain + bid + nid * int(Nstride)) % Nblocks] for bid in range(Nvalid) ]) if set_idx.size != 0: data_count[nid, 1] += set_idx.size cache.insertdata(X[set_idx], F[set_idx], E[set_idx], list(S)) if build_valid: valdh5[nid].store_data(f + data['path'], coordinates=X[set_idx], forces=F[set_idx], energies=E[set_idx], species=list(S)) if build_test: for nid, th5 in enumerate(testh5): set_idx = np.concatenate([ Didx[(Ntrain + Nvalid + bid + nid * int(Nstride)) % Nblocks] for bid in range(Ntest) ]) if set_idx.size != 0: data_count[nid, 2] += set_idx.size th5.store_data(f + data['path'], coordinates=X[set_idx], forces=F[set_idx], energies=E[set_idx], species=list(S)) # Save train and valid meta file and cleanup testh5 for t, v in zip(cachet, cachev): t.makemetadata() v.makemetadata() if build_test: for th in testh5: th.cleanup() if build_valid: for vh in valdh5: vh.cleanup() print(' Train ', ' Valid ', ' Test ') print(data_count) print('Training set built.')
def build_training_cache(self, forces=True): store_dir = self.train_root + "cache-data-" N = self.Nn for i in range(N): if not os.path.exists(store_dir + str(i)): os.mkdir(store_dir + str(i)) if os.path.exists(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5'): os.remove(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5') if not os.path.exists(store_dir + str(i) + '/../testset'): os.mkdir(store_dir + str(i) + '/../testset') cachet = [ cg('_train', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] cachev = [ cg('_valid', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] testh5 = [ pyt.datapacker(store_dir + str(r) + '/../testset/testset' + str(r) + '.h5') for r in range(N) ] Nd = np.zeros(N, dtype=np.int32) Nbf = 0 for f, fn in enumerate(self.h5file): print( 'Processing file(' + str(f + 1) + ' of ' + str(len(self.h5file)) + '):', fn) adl = pyt.anidataloader(self.h5dir + fn) To = adl.size() Ndc = 0 Fmt = [] Emt = [] for c, data in enumerate(adl): Pn = data['path'] + '_' + str(f).zfill(6) + '_' + str(c).zfill( 6) # Extract the data X = data['coordinates'] E = data['energies'] S = data['species'] # 0.0 forces if key doesnt exist if forces: F = data['forces'] else: F = 0.0 * X Fmt.append(np.max(np.linalg.norm(F, axis=2), axis=1)) Emt.append(E) Mv = np.max(np.linalg.norm(F, axis=2), axis=1) index = np.where(Mv > 10.5)[0] indexk = np.where(Mv <= 10.5)[0] Nbf += index.size # Clear forces X = X[indexk] F = F[indexk] E = E[indexk] Esae = hdt.compute_sae(self.netdict['saefile'], S) hidx = np.where(np.abs(E - Esae) > 10.0) lidx = np.where(np.abs(E - Esae) <= 10.0) if hidx[0].size > 0: print( ' -(' + str(c).zfill(3) + ')High energies detected:\n ', E[hidx]) X = X[lidx] E = E[lidx] F = F[lidx] Ndc += E.size if (set(S).issubset(self.netdict['atomtyp'])): # Random mask R = np.random.uniform(0.0, 1.0, E.shape[0]) idx = np.array([interval(r, N) for r in R]) # Build random split lists split = [] for j in range(N): split.append([i for i, s in enumerate(idx) if s == j]) nd = len([i for i, s in enumerate(idx) if s == j]) Nd[j] = Nd[j] + nd # Store data for i, t, v, te in zip(range(N), cachet, cachev, testh5): ## Store training data X_t = np.array(np.concatenate( [X[s] for j, s in enumerate(split) if j != i]), order='C', dtype=np.float32) F_t = np.array(np.concatenate( [F[s] for j, s in enumerate(split) if j != i]), order='C', dtype=np.float32) E_t = np.array(np.concatenate( [E[s] for j, s in enumerate(split) if j != i]), order='C', dtype=np.float64) if E_t.shape[0] != 0: t.insertdata(X_t, F_t, E_t, list(S)) ## Store Validation if np.array(split[i]).size > 0: X_v = np.array(X[split[i]], order='C', dtype=np.float32) F_v = np.array(F[split[i]], order='C', dtype=np.float32) E_v = np.array(E[split[i]], order='C', dtype=np.float64) if E_v.shape[0] != 0: v.insertdata(X_v, F_v, E_v, list(S)) # Print some stats print('Data count:', Nd) print('Data split:', 100.0 * Nd / np.sum(Nd), '%') # Save train and valid meta file and cleanup testh5 for t, v, th in zip(cachet, cachev, testh5): t.makemetadata() v.makemetadata() th.cleanup()
N = 5 for i in range(N): if not os.path.exists(store_dir + str(i)): os.mkdir(store_dir + str(i)) if os.path.exists(wkdir + 'testset.h5'): os.remove(wkdir + 'testset.h5') cachet = [ cg('_train', saef, store_dir + str(r) + '/', False) for r in range(N) ] cachev = [ cg('_valid', saef, store_dir + str(r) + '/', False) for r in range(N) ] testh5 = pyt.datapacker(wkdir + 'testset.h5') Nd = np.zeros(N, dtype=np.int32) Nbf = 0 for f, fn in enumerate(h5files): print('Processing file(' + str(f + 1) + ' of ' + str(len(h5files)) + '):', fn[1]) adl = pyt.anidataloader(fn) To = adl.size() Ndc = 0 Fmt = [] Emt = [] for c, data in enumerate(adl): #if c == 2 or c == 2 or c == 2: # Get test store name
# Construct pyNeuroChem classes print('Constructing CV network list...') ncl = [pync.conformers(cnstfile, saefile, wkdir + 'cv_train_' + str(l) + '/networks/', 0, False) for l in range(4)] print('Complete.') store_xyz = '/home/jujuman/Research/DataReductionMethods/models/cv/bad_xyz/' svpath = '/home/jujuman/Scratch/Research/DataReductionMethods/models/ani_red_cnl_c08f.h5' h5file = '/home/jujuman/Scratch/Research/DataReductionMethods/models/train_c08f/ani_red_c08f.h5' # Remove file if exists if os.path.exists(svpath): os.remove(svpath) #open an HDF5 for compressed storage. dpack = pyt.datapacker(svpath) # Declare loader adl = pyt.anidataloader(h5file) Nd = 0 Nb = 0 for data in adl: # Extract the data Ea = data['energies'] S = data['species'] X = data['coordinates'].reshape(Ea.shape[0], len(S),3) El = [] for nc in ncl: nc.setConformers(confs=X, types=list(S))
N = 5 for i in range(N): if not os.path.exists(store_dir + str(i)): os.mkdir(store_dir + str(i)) if os.path.exists(store_dir + str(i) + '/../testset/testset'+str(i)+'.h5'): os.remove(store_dir + str(i) + '/../testset/testset'+str(i)+'.h5') if not os.path.exists(store_dir + str(i) + '/../testset'): os.mkdir(store_dir + str(i) + '/../testset') cachet = [cg('_train', saef, store_dir + str(r) + '/',False) for r in range(N)] cachev = [cg('_valid', saef, store_dir + str(r) + '/',False) for r in range(N)] testh5 = [pyt.datapacker(store_dir + str(r) + '/../testset/testset'+str(r)+'.h5') for r in range(N)] Nd = np.zeros(N,dtype=np.int32) Nbf = 0 for f,fn in enumerate(h5files): print('Processing file('+ str(f+1) +' of '+ str(len(h5files)) +'):', fn) adl = pyt.anidataloader(fn) To = adl.size() Ndc = 0 Fmt = [] Emt = [] for c, data in enumerate(adl): #if c == 2 or c == 2 or c == 2: # Get test store name #Pn = fn.split('/')[-1].split('.')[0] + data['path']
if os.path.exists(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5'): os.remove(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5') if not os.path.exists(store_dir + str(i) + '/../testset'): os.mkdir(store_dir + str(i) + '/../testset') cachet = [ cg('_train', saef, store_dir + str(r) + '/', False) for r in range(N) ] cachev = [ cg('_valid', saef, store_dir + str(r) + '/', False) for r in range(N) ] testh5 = [ pyt.datapacker(store_dir + str(r) + '/../testset/testset' + str(r) + '.h5') for r in range(N) ] Nd = np.zeros(N, dtype=np.int32) Nbf = 0 for f, fn in enumerate(h5files): print('Processing file(' + str(f + 1) + ' of ' + str(len(h5files)) + '):', fn) adl = pyt.anidataloader(fn) To = adl.size() Ndc = 0 Fmt = [] Emt = [] for c, data in enumerate(adl):
def build_strided_training_cache(self, Nblocks, Nvalid, Ntest, build_test=True, forces=True, grad=False, Fkey='forces', forces_unit=1.0, Ekey='energies', energy_unit=1.0, Eax0sum=False): if not os.path.isfile(self.netdict['saefile']): self.sae_linear_fitting(Ekey=Ekey, energy_unit=energy_unit, Eax0sum=Eax0sum) h5d = self.h5dir store_dir = self.train_root + "cache-data-" N = self.Nn Ntrain = Nblocks - Nvalid - Ntest if Nblocks % N != 0: raise ValueError( 'Error: number of networks must evenly divide number of blocks.' ) Nstride = Nblocks / N for i in range(N): if not os.path.exists(store_dir + str(i)): os.mkdir(store_dir + str(i)) if build_test: if os.path.exists(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5'): os.remove(store_dir + str(i) + '/../testset/testset' + str(i) + '.h5') if not os.path.exists(store_dir + str(i) + '/../testset'): os.mkdir(store_dir + str(i) + '/../testset') cachet = [ cg('_train', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] cachev = [ cg('_valid', self.netdict['saefile'], store_dir + str(r) + '/', False) for r in range(N) ] if build_test: testh5 = [ pyt.datapacker(store_dir + str(r) + '/../testset/testset' + str(r) + '.h5') for r in range(N) ] E = [] data_count = np.zeros((N, 3), dtype=np.int32) for f in self.h5file: adl = pyt.anidataloader(h5d + f) for data in adl: #print(data['path'],data['energies'].size) S = data['species'] if data[Ekey].size > 0 and (set(S).issubset( self.netdict['atomtyp'])): X = np.array(data['coordinates'], order='C', dtype=np.float32) if Eax0sum: E = energy_unit * np.sum(np.array( data[Ekey], order='C', dtype=np.float64), axis=1) else: E = energy_unit * np.array( data[Ekey], order='C', dtype=np.float64) if forces and not grad: F = forces_unit * np.array( data[Fkey], order='C', dtype=np.float32) if forces and grad: F = -forces_unit * np.array( data[Fkey], order='C', dtype=np.float32) else: F = 0.0 * X # Build random split index ridx = np.random.randint(0, Nblocks, size=E.size) Didx = [ np.argsort(ridx)[np.where(ridx == i)] for i in range(Nblocks) ] # Build training cache for nid, cache in enumerate(cachet): set_idx = np.concatenate([ Didx[((bid + nid * int(Nstride)) % Nblocks)] for bid in range(Ntrain) ]) if set_idx.size != 0: data_count[nid, 0] += set_idx.size cache.insertdata(X[set_idx], F[set_idx], E[set_idx], list(S)) for nid, cache in enumerate(cachev): set_idx = np.concatenate([ Didx[(Ntrain + bid + nid * int(Nstride)) % Nblocks] for bid in range(Nvalid) ]) if set_idx.size != 0: data_count[nid, 1] += set_idx.size cache.insertdata(X[set_idx], F[set_idx], E[set_idx], list(S)) if build_test: for nid, th5 in enumerate(testh5): set_idx = np.concatenate([ Didx[(Ntrain + Nvalid + bid + nid * int(Nstride)) % Nblocks] for bid in range(Ntest) ]) if set_idx.size != 0: data_count[nid, 2] += set_idx.size th5.store_data(f + data['path'], coordinates=X[set_idx], forces=F[set_idx], energies=E[set_idx], species=list(S)) # Save train and valid meta file and cleanup testh5 for t, v in zip(cachet, cachev): t.makemetadata() v.makemetadata() if build_test: for th in testh5: th.cleanup() print(' Train ', ' Valid ', ' Test ') print(data_count) print('Training set built.')
def __init__(self, hdf5files, saef, storecac, storetest): self.xyz = [] self.Eqm = [] self.spc = [] self.idx = [] self.prt = [] self.kid = [] # list to track data kept self.nt = [] # total conformers self.nc = [] # total kept self.tf = 0 for f in hdf5files: # Construct the data loader class adl = pyt.anidataloader(f) # Declare test cache if os.path.exists(storetest): os.remove(storetest) dpack = pyt.datapacker(storetest) for i, data in enumerate(adl): xyz = np.array_split(data['coordinates'], 10) eng = np.array_split(data['energies'], 10) spc = data['species'] nme = data['parent'] self.prt.append(nme) self.xyz.append( np.concatenate(xyz[0:9]) ) self.Eqm.append( np.concatenate(eng[0:9]) ) self.spc.append(spc) Nd = np.concatenate(eng[0:9]).shape[0] self.idx.append( np.arange(Nd) ) self.kid.append( np.array([], dtype=np.int) ) self.tf = self.tf + Nd self.nt.append(Nd) self.nc.append(0) # Prepare and store the test data set if xyz[9].size != 0: t_xyz = xyz[9].reshape(xyz[9].shape[0], xyz[9].shape[1] * xyz[9].shape[2]) dpack.store_data(nme + '/mol' + str(i), coordinates=t_xyz, energies=np.array(eng[9]), species=spc) # Clean up adl.cleanup() # Clean up dpack.cleanup() self.nt = np.array(self.nt) self.nc = np.array(self.nc) self.ts = 0 self.vs = 0 self.Nbad = self.tf self.saef = saef self.storecac = storecac
import numpy as np import hdnntools as gt import pyanitools as pyt import os lfile = '/home/jujuman/DataTesting/gdb9-2500-div-dim.h5' sfile = '/home/jujuman/DataTesting/gdb9-2500-div-dim_35.h5' if os.path.exists(sfile): os.remove(sfile) adl = pyt.anidataloader(lfile) dpk = pyt.datapacker(sfile) for i,x in enumerate(adl): print(i) xyz = np.asarray(x['coordinates'],dtype=np.float32) erg = x['energies'] spc = x['species'] dpk.store_data('/gdb-09-DIV/mol'+str(i), coordinates=xyz.reshape(erg.shape[0],len(spc)*3), energies=erg, species=spc) adl.cleanup() dpk.cleanup()
import hdnntools as hdt def check_for_outsider(okayl, chckl): for i in chckl: if i not in okayl: return False return True dst = "/home/jujuman/Research/ANI-DATASET/h5data/gdb9-2500-bad_new.h5" src = "/home/jujuman/Research/ANI-DATASET/GDB-09-Data/gdb9-2500-bad.h5" #open an HDF5 for compressed storage. #Note that if the path exists, it will open whatever is there. dpack = pyt.datapacker(dst) aload = pyt.anidataloader(src) at = [ 'H', 'C', 'N', 'O', #'F', #'S', ] for id, data in enumerate(aload.get_roman_data()): xyz = np.asarray(data['coordinates'], dtype=np.float32) erg = np.asarray(data['energies'], dtype=np.float64)
import pyanitools as pyt #import pyaniasetools as aat import numpy as np import hdnntools as hdt import os #import matplotlib.pyplot as plt file_old = '/home/jsmith48/scratch/auto_al/h5files/ANI-AL-0707.0000.0408.h5' file_new = '/home/jsmith48/scratch/auto_al/h5files_fix/ANI-AL-0707.0000.0408.h5' print('Working on file:', file_old) adl = pyt.anidataloader(file_old) # Data storage dpack = pyt.datapacker(file_new, mode='w') for i, data in enumerate(adl): #if i == 20: # break X = data['coordinates'] S = data['species'] Edft = data['energies'] path = data['path'] del data['path'] #Eani, Fani = anicv.compute_energy_conformations(X=np.array(X,dtype=np.float32),S=S) Esae = hdt.compute_sae( '/home/jsmith48/scratch/auto_al/modelCNOSFCl/sae_wb97x-631gd.dat', S)