def extend_test():
import numpy as np
import pyemu
first = pyemu.Cov(x=np.ones((3,3))+1.0,names=["p1","p2","p3"],isdiagonal=False)
second = pyemu.Cov(x=np.ones((4,1))+2.0,names=["o1","o2","o3","o4"],isdiagonal=True)
third = first.extend(second)
print(third)
assert third.x[:,0].sum() == 6
assert third.x[0,:].sum() == 6
assert third.x[:,2].sum() == 6
assert third.x[2,:].sum() == 6
assert third.x[:,3].sum() == 3
assert third.x[3,:].sum() == 3
assert third.x[:,6].sum() == 3
assert third.x[6,:].sum() == 3
try:
forth = pyemu.mat.concat([first,third])
except:
pass
else:
raise Exception()
forth = pyemu.Matrix(x=first.x,row_names=first.row_names,col_names=[str(i) for i in range(first.shape[1])])
x = pyemu.mat.concat([first,forth])
print(x)
fifth = pyemu.Matrix(x=first.x, row_names=[str(i) for i in range(first.shape[0])], col_names=first.col_names)
x = pyemu.mat.concat([first,fifth])
print(x)
def sparse_extend_test():
import os
from datetime import datetime
import numpy as np
import pyemu
nrow = 5
ncol = 5
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm = pyemu.SparseMatrix.from_matrix(m)
try:
sm.block_extend_ip(m)
except:
pass
else:
raise Exception()
m = pyemu.Matrix(x,
row_names=['t{0}'.format(i) for i in range(nrow)],
col_names=m.col_names)
try:
sm.block_extend_ip(m)
except:
pass
else:
raise Exception()
m = pyemu.Matrix(x,
row_names=['r{0}'.format(i) for i in range(nrow)],
col_names=['r{0}'.format(i) for i in range(ncol)])
sm.block_extend_ip(m)
m1 = sm.to_matrix()
d = m.x - m1.x[m.shape[0]:, m.shape[1]:]
assert d.sum() == 0
m = pyemu.Cov(x=np.atleast_2d(np.ones(nrow)),
names=['p{0}'.format(i) for i in range(nrow)],
isdiagonal=True)
sm.block_extend_ip(m)
d = m.as_2d - sm.to_matrix().x[-nrow:, -nrow:]
assert d.sum() == 0
m1 = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm1 = pyemu.SparseMatrix.from_matrix(m1)
sm2 = pyemu.SparseMatrix.from_matrix(m)
sm1.block_extend_ip(sm2)
m2 = sm1.to_matrix()
d = m.as_2d - m2.x[m.shape[0]:, m.shape[1]:]
assert d.sum() == 0
def hadamard_product_test():
import os
import numpy as np
import pyemu
jco = pyemu.Jco.from_binary(os.path.join("mat", "pest.jcb"))
z = jco.zero2d
#print(jco.shape)
#print(z.shape)
hp = jco.hadamard_product(z)
assert hp.x.sum() == 0.0
hp = z.hadamard_product(hp)
assert hp.x.sum() == 0.0
c = pyemu.Cov(x=np.ones((jco.shape[0], 1)),
names=jco.row_names,
isdiagonal=True)
r = pyemu.Matrix(x=np.random.rand(c.shape[0], c.shape[0]),
row_names=c.row_names,
col_names=c.col_names)
hp = c.hadamard_product(r)
assert hp.x.sum() == np.diagonal(r.x).sum()
hp = r.hadamard_product(c)
assert hp.x.sum() == np.diagonal(r.x).sum()
def sparse_constructor_test():
import os
from datetime import datetime
import numpy as np
import pyemu
nrow = 100
ncol = 100
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm = pyemu.SparseMatrix.from_matrix(m)
mname = os.path.join("temp","test.jcb")
m.to_binary(mname)
sm = pyemu.SparseMatrix.from_binary(mname)
sm.to_coo(mname)
m1 = sm.to_matrix()
m = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m1.x,m.x)
def coo_tests():
import os
from datetime import datetime
import numpy as np
import pyemu
nrow = 100
ncol = 1000
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
assert m.shape[0] == len(rnames)
assert m.shape[1] == len(cnames)
mname = os.path.join("temp", "temp.jcb")
m.to_coo(mname)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_coo(mname, chunk=1)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_coo(mname, chunk=100000)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_coo(mname, chunk=1000)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_binary(mname)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_binary(mname)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_binary(mname)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
m.to_binary(mname)
mm = pyemu.Matrix.from_binary(mname)
assert np.array_equal(m.x, mm.x)
os.remove(mname)
def get_diag_test():
import numpy as np
import pyemu
n = 100
col_names = ["cname_{0}".format(i) for i in range(n)]
row_names = ["rname_{0}".format(i) for i in range(n)]
arr = np.random.random(n * n)
arr.resize((n, n))
mat = pyemu.Matrix(x=arr, row_names=row_names, col_names=col_names)
diag_mat = mat.get_diagonal_vector(col_name="test")
s1 = np.diag(arr).sum()
s2 = diag_mat.x.sum()
assert s1 == s2
def sparse_get_sparse_test():
import os
from datetime import datetime
import numpy as np
import pyemu
nrow = 5
ncol = 5
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm = pyemu.SparseMatrix.from_matrix(m)
m1 = sm.get_matrix(rnames[0],cnames)
d = m1.x - m.x[0,:]
assert d.sum() == 0
sm = pyemu.SparseMatrix.from_matrix(m)
m1 = sm.get_sparse_matrix(rnames[:2], cnames).to_matrix()
d = m1.x - m.x[:2, :]
assert d.sum() == 0
sm = pyemu.SparseMatrix.from_matrix(m)
m1 = sm.get_sparse_matrix(rnames, cnames[0]).to_matrix()
d = m1.x - m.x[:, 0]
assert d.sum() == 0
sm = pyemu.SparseMatrix.from_matrix(m)
m1 = sm.get_sparse_matrix(rnames, cnames[:2]).to_matrix()
d = m1.x - m.x[:, :2]
assert d.sum() == 0
sm = pyemu.SparseMatrix.from_matrix(m)
m1 = sm.get_sparse_matrix(rnames, cnames).to_matrix()
d = m1.x - m.x
assert d.sum() == 0
def df_tests():
import os
import numpy as np
import pandas as pd
import pyemu
nrow = 5
ncol = 5
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
df = pd.DataFrame(data=x,columns=cnames,index=rnames)
#sub
d = m - df
assert d.x.max() == 0.0
d = df - m.x #returns a df
#print(d.max())
# add
d = (m + df) - (df * 2)
assert d.x.max() == 0.0
d = (df * 2) - (m + df).x #returns a df
# mul
d = (m * df.T) - np.dot(m.x,df.T.values)
assert d.x.max() == 0.0
# hadamard
d = (m.hadamard_product(df)) - (m.x * df)
assert d.x.max() == 0.0
def kl_setup(num_eig,
sr,
struct_file,
array_dict,
basis_file="basis.dat",
tpl_file="kl.tpl"):
"""setup a karhuenen-Loeve based parameterization for a given
geostatistical structure.
Parameters:
num_eig (int) : number of basis vectors to retain in the reduced basis
struct_file (str) : a pest-style geostatistical structure file
array_dict (dict(str:ndarray)): a dict of arrays to setup as KL-based
parameters. The key becomes the
parameter name prefix. The total
number of parameters is
len(array_dict) * num_eig
basis_file (str) : the name of the binary file where the reduced
basis will be saved
tpl_file (str) : the name of the template file to make. The template
file is a csv file with the parameter names, the
original factor values,and the template entries.
The original values can be used to set the parval1
entries in the control file
Returns:
back_array_dict (dict(str:ndarray)) : a dictionary of back transformed
arrays. This is useful to see
how much "smoothing" is taking
place compared to the original
arrays.
"""
try:
import flopy
except Exception as e:
raise Exception("error import flopy: {0}".format(str(e)))
assert isinstance(sr, flopy.utils.SpatialReference)
for name, array in array_dict.items():
assert isinstance(array, np.ndarray)
assert array.shape[0] == sr.nrow
assert array.shape[1] == sr.ncol
assert len(name) + len(str(num_eig)) <= 12,"name too long:{0}".\
format(name)
assert os.path.exists(struct_file)
gs = pyemu.utils.read_struct_file(struct_file)
names = []
for i in range(sr.nrow):
names.extend(["i{0:04d}j{1:04d}".format(i, j) for j in range(sr.ncol)])
cov = gs.covariance_matrix(sr.xcentergrid.flatten(),
sr.ycentergrid.flatten(),
names=names)
trunc_basis = cov.u[:, :num_eig].T
#for i in range(num_eig):
# trunc_basis.x[i,:] *= cov.s.x[i]
trunc_basis.to_binary(basis_file)
#trunc_basis = trunc_basis.T
back_array_dict = {}
f = open(tpl_file, 'w')
f.write("ptf ~\n")
f.write("name,org_val,new_val\n")
for name, array in array_dict.items():
mname = name + "mean"
f.write("{0},{1:20.8E},~ {2} ~\n".format(mname, 0.0, mname))
#array -= array.mean()
array_flat = pyemu.Matrix(x=np.atleast_2d(array.flatten()).transpose(),
col_names=["flat"],
row_names=names,
isdiagonal=False)
factors = trunc_basis * array_flat
enames = ["{0}{1:04d}".format(name, i) for i in range(num_eig)]
for n, val in zip(enames, factors.x):
f.write("{0},{1:20.8E},~ {0} ~\n".format(n, val[0]))
back_array_dict[name] = (factors.T * trunc_basis).x.reshape(
array.shape)
#print(array_back)
#print(factors.shape)
return back_array_dict
def dense_mat_format_test():
import numpy as np
import pyemu
from datetime import datetime
nrow = 100
ncol = 500000
long_str = ""
for _ in range(35):
long_str += "long"
rnames = [long_str + "row_{0}".format(i) for i in range(nrow)]
cnames = [long_str + "col_{0}".format(i) for i in range(ncol)]
arr = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=arr, row_names=rnames, col_names=cnames)
f = m.to_dense("dense.bin", close=True)
m1 = pyemu.Matrix.from_binary("dense.bin")
print(m1.shape)
assert m1.shape == (nrow, ncol)
d = np.abs(m.x - m1.x).sum()
print(d)
assert d < 1.0e-10
f_in = open("dense.bin", "rb")
f_out = open("dense_trunc.bin", "wb")
f_out.write(f_in.read(ncol * (len(long_str)) + int(nrow / 2) * ncol * 8))
f_in.close()
f_out.close()
try:
m1 = pyemu.Matrix.from_binary("dense_trunc.bin", forgive=False)
except:
pass
else:
raise Exception("should have failed")
m1 = pyemu.Matrix.from_binary("dense_trunc.bin", forgive=True)
m = pyemu.Matrix(x=arr, row_names=rnames, col_names=cnames)
f = m.to_dense("dense.bin", close=False)
new_rnames = [r + "new" for r in rnames]
m.row_names = new_rnames
m.to_dense(f, close=True)
m1 = pyemu.Matrix.from_binary("dense.bin")
print(m1.shape)
assert m1.shape == (nrow * 2, ncol)
arr2 = np.zeros((nrow * 2, ncol))
arr2[:nrow, :] = arr
arr2[nrow:, :] = arr
d = np.abs(arr2 - m1.x).sum()
print(d)
assert d < 1.0e-10, d
s1 = datetime.now()
for _ in range(1):
m.to_dense("dense.bin")
pyemu.Matrix.read_binary("dense.bin")
e1 = datetime.now()
s2 = datetime.now()
for _ in range(1):
m.to_coo("dense.jcb")
pyemu.Matrix.read_binary("dense.jcb")
e2 = datetime.now()
print((e1 - s1).total_seconds())
print((e2 - s2).total_seconds())
