def extend_test(): import numpy as np import pyemu first = pyemu.Cov(x=np.ones((3,3))+1.0,names=["p1","p2","p3"],isdiagonal=False) second = pyemu.Cov(x=np.ones((4,1))+2.0,names=["o1","o2","o3","o4"],isdiagonal=True) third = first.extend(second) print(third) assert third.x[:,0].sum() == 6 assert third.x[0,:].sum() == 6 assert third.x[:,2].sum() == 6 assert third.x[2,:].sum() == 6 assert third.x[:,3].sum() == 3 assert third.x[3,:].sum() == 3 assert third.x[:,6].sum() == 3 assert third.x[6,:].sum() == 3 try: forth = pyemu.mat.concat([first,third]) except: pass else: raise Exception() forth = pyemu.Matrix(x=first.x,row_names=first.row_names,col_names=[str(i) for i in range(first.shape[1])]) x = pyemu.mat.concat([first,forth]) print(x) fifth = pyemu.Matrix(x=first.x, row_names=[str(i) for i in range(first.shape[0])], col_names=first.col_names) x = pyemu.mat.concat([first,fifth]) print(x)
def sparse_extend_test(): import os from datetime import datetime import numpy as np import pyemu nrow = 5 ncol = 5 rnames = ["row_{0}".format(i) for i in range(nrow)] cnames = ["col_{0}".format(i) for i in range(ncol)] x = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) sm = pyemu.SparseMatrix.from_matrix(m) try: sm.block_extend_ip(m) except: pass else: raise Exception() m = pyemu.Matrix(x, row_names=['t{0}'.format(i) for i in range(nrow)], col_names=m.col_names) try: sm.block_extend_ip(m) except: pass else: raise Exception() m = pyemu.Matrix(x, row_names=['r{0}'.format(i) for i in range(nrow)], col_names=['r{0}'.format(i) for i in range(ncol)]) sm.block_extend_ip(m) m1 = sm.to_matrix() d = m.x - m1.x[m.shape[0]:, m.shape[1]:] assert d.sum() == 0 m = pyemu.Cov(x=np.atleast_2d(np.ones(nrow)), names=['p{0}'.format(i) for i in range(nrow)], isdiagonal=True) sm.block_extend_ip(m) d = m.as_2d - sm.to_matrix().x[-nrow:, -nrow:] assert d.sum() == 0 m1 = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) sm1 = pyemu.SparseMatrix.from_matrix(m1) sm2 = pyemu.SparseMatrix.from_matrix(m) sm1.block_extend_ip(sm2) m2 = sm1.to_matrix() d = m.as_2d - m2.x[m.shape[0]:, m.shape[1]:] assert d.sum() == 0
def hadamard_product_test(): import os import numpy as np import pyemu jco = pyemu.Jco.from_binary(os.path.join("mat", "pest.jcb")) z = jco.zero2d #print(jco.shape) #print(z.shape) hp = jco.hadamard_product(z) assert hp.x.sum() == 0.0 hp = z.hadamard_product(hp) assert hp.x.sum() == 0.0 c = pyemu.Cov(x=np.ones((jco.shape[0], 1)), names=jco.row_names, isdiagonal=True) r = pyemu.Matrix(x=np.random.rand(c.shape[0], c.shape[0]), row_names=c.row_names, col_names=c.col_names) hp = c.hadamard_product(r) assert hp.x.sum() == np.diagonal(r.x).sum() hp = r.hadamard_product(c) assert hp.x.sum() == np.diagonal(r.x).sum()
def sparse_constructor_test(): import os from datetime import datetime import numpy as np import pyemu nrow = 100 ncol = 100 rnames = ["row_{0}".format(i) for i in range(nrow)] cnames = ["col_{0}".format(i) for i in range(ncol)] x = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) sm = pyemu.SparseMatrix.from_matrix(m) mname = os.path.join("temp","test.jcb") m.to_binary(mname) sm = pyemu.SparseMatrix.from_binary(mname) sm.to_coo(mname) m1 = sm.to_matrix() m = pyemu.Matrix.from_binary(mname) assert np.array_equal(m1.x,m.x)
def coo_tests(): import os from datetime import datetime import numpy as np import pyemu nrow = 100 ncol = 1000 rnames = ["row_{0}".format(i) for i in range(nrow)] cnames = ["col_{0}".format(i) for i in range(ncol)] x = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) assert m.shape[0] == len(rnames) assert m.shape[1] == len(cnames) mname = os.path.join("temp", "temp.jcb") m.to_coo(mname) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_coo(mname, chunk=1) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_coo(mname, chunk=100000) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_coo(mname, chunk=1000) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_binary(mname) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_binary(mname) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_binary(mname) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname) m.to_binary(mname) mm = pyemu.Matrix.from_binary(mname) assert np.array_equal(m.x, mm.x) os.remove(mname)
def get_diag_test(): import numpy as np import pyemu n = 100 col_names = ["cname_{0}".format(i) for i in range(n)] row_names = ["rname_{0}".format(i) for i in range(n)] arr = np.random.random(n * n) arr.resize((n, n)) mat = pyemu.Matrix(x=arr, row_names=row_names, col_names=col_names) diag_mat = mat.get_diagonal_vector(col_name="test") s1 = np.diag(arr).sum() s2 = diag_mat.x.sum() assert s1 == s2
def sparse_get_sparse_test(): import os from datetime import datetime import numpy as np import pyemu nrow = 5 ncol = 5 rnames = ["row_{0}".format(i) for i in range(nrow)] cnames = ["col_{0}".format(i) for i in range(ncol)] x = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) sm = pyemu.SparseMatrix.from_matrix(m) m1 = sm.get_matrix(rnames[0],cnames) d = m1.x - m.x[0,:] assert d.sum() == 0 sm = pyemu.SparseMatrix.from_matrix(m) m1 = sm.get_sparse_matrix(rnames[:2], cnames).to_matrix() d = m1.x - m.x[:2, :] assert d.sum() == 0 sm = pyemu.SparseMatrix.from_matrix(m) m1 = sm.get_sparse_matrix(rnames, cnames[0]).to_matrix() d = m1.x - m.x[:, 0] assert d.sum() == 0 sm = pyemu.SparseMatrix.from_matrix(m) m1 = sm.get_sparse_matrix(rnames, cnames[:2]).to_matrix() d = m1.x - m.x[:, :2] assert d.sum() == 0 sm = pyemu.SparseMatrix.from_matrix(m) m1 = sm.get_sparse_matrix(rnames, cnames).to_matrix() d = m1.x - m.x assert d.sum() == 0
def df_tests(): import os import numpy as np import pandas as pd import pyemu nrow = 5 ncol = 5 rnames = ["row_{0}".format(i) for i in range(nrow)] cnames = ["col_{0}".format(i) for i in range(ncol)] x = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames) df = pd.DataFrame(data=x,columns=cnames,index=rnames) #sub d = m - df assert d.x.max() == 0.0 d = df - m.x #returns a df #print(d.max()) # add d = (m + df) - (df * 2) assert d.x.max() == 0.0 d = (df * 2) - (m + df).x #returns a df # mul d = (m * df.T) - np.dot(m.x,df.T.values) assert d.x.max() == 0.0 # hadamard d = (m.hadamard_product(df)) - (m.x * df) assert d.x.max() == 0.0
def kl_setup(num_eig, sr, struct_file, array_dict, basis_file="basis.dat", tpl_file="kl.tpl"): """setup a karhuenen-Loeve based parameterization for a given geostatistical structure. Parameters: num_eig (int) : number of basis vectors to retain in the reduced basis struct_file (str) : a pest-style geostatistical structure file array_dict (dict(str:ndarray)): a dict of arrays to setup as KL-based parameters. The key becomes the parameter name prefix. The total number of parameters is len(array_dict) * num_eig basis_file (str) : the name of the binary file where the reduced basis will be saved tpl_file (str) : the name of the template file to make. The template file is a csv file with the parameter names, the original factor values,and the template entries. The original values can be used to set the parval1 entries in the control file Returns: back_array_dict (dict(str:ndarray)) : a dictionary of back transformed arrays. This is useful to see how much "smoothing" is taking place compared to the original arrays. """ try: import flopy except Exception as e: raise Exception("error import flopy: {0}".format(str(e))) assert isinstance(sr, flopy.utils.SpatialReference) for name, array in array_dict.items(): assert isinstance(array, np.ndarray) assert array.shape[0] == sr.nrow assert array.shape[1] == sr.ncol assert len(name) + len(str(num_eig)) <= 12,"name too long:{0}".\ format(name) assert os.path.exists(struct_file) gs = pyemu.utils.read_struct_file(struct_file) names = [] for i in range(sr.nrow): names.extend(["i{0:04d}j{1:04d}".format(i, j) for j in range(sr.ncol)]) cov = gs.covariance_matrix(sr.xcentergrid.flatten(), sr.ycentergrid.flatten(), names=names) trunc_basis = cov.u[:, :num_eig].T #for i in range(num_eig): # trunc_basis.x[i,:] *= cov.s.x[i] trunc_basis.to_binary(basis_file) #trunc_basis = trunc_basis.T back_array_dict = {} f = open(tpl_file, 'w') f.write("ptf ~\n") f.write("name,org_val,new_val\n") for name, array in array_dict.items(): mname = name + "mean" f.write("{0},{1:20.8E},~ {2} ~\n".format(mname, 0.0, mname)) #array -= array.mean() array_flat = pyemu.Matrix(x=np.atleast_2d(array.flatten()).transpose(), col_names=["flat"], row_names=names, isdiagonal=False) factors = trunc_basis * array_flat enames = ["{0}{1:04d}".format(name, i) for i in range(num_eig)] for n, val in zip(enames, factors.x): f.write("{0},{1:20.8E},~ {0} ~\n".format(n, val[0])) back_array_dict[name] = (factors.T * trunc_basis).x.reshape( array.shape) #print(array_back) #print(factors.shape) return back_array_dict
def dense_mat_format_test(): import numpy as np import pyemu from datetime import datetime nrow = 100 ncol = 500000 long_str = "" for _ in range(35): long_str += "long" rnames = [long_str + "row_{0}".format(i) for i in range(nrow)] cnames = [long_str + "col_{0}".format(i) for i in range(ncol)] arr = np.random.random((nrow, ncol)) m = pyemu.Matrix(x=arr, row_names=rnames, col_names=cnames) f = m.to_dense("dense.bin", close=True) m1 = pyemu.Matrix.from_binary("dense.bin") print(m1.shape) assert m1.shape == (nrow, ncol) d = np.abs(m.x - m1.x).sum() print(d) assert d < 1.0e-10 f_in = open("dense.bin", "rb") f_out = open("dense_trunc.bin", "wb") f_out.write(f_in.read(ncol * (len(long_str)) + int(nrow / 2) * ncol * 8)) f_in.close() f_out.close() try: m1 = pyemu.Matrix.from_binary("dense_trunc.bin", forgive=False) except: pass else: raise Exception("should have failed") m1 = pyemu.Matrix.from_binary("dense_trunc.bin", forgive=True) m = pyemu.Matrix(x=arr, row_names=rnames, col_names=cnames) f = m.to_dense("dense.bin", close=False) new_rnames = [r + "new" for r in rnames] m.row_names = new_rnames m.to_dense(f, close=True) m1 = pyemu.Matrix.from_binary("dense.bin") print(m1.shape) assert m1.shape == (nrow * 2, ncol) arr2 = np.zeros((nrow * 2, ncol)) arr2[:nrow, :] = arr arr2[nrow:, :] = arr d = np.abs(arr2 - m1.x).sum() print(d) assert d < 1.0e-10, d s1 = datetime.now() for _ in range(1): m.to_dense("dense.bin") pyemu.Matrix.read_binary("dense.bin") e1 = datetime.now() s2 = datetime.now() for _ in range(1): m.to_coo("dense.jcb") pyemu.Matrix.read_binary("dense.jcb") e2 = datetime.now() print((e1 - s1).total_seconds()) print((e2 - s2).total_seconds())