def test_var_bounds(self):
""" Test bounds on optimisation variables.
"""
msg = self.case_name
_, xmin, xmax = self.solver._var_bounds()
mpxmin = mmread(join(DATA_DIR, self.case_name, "opf", "xmin_AC.mtx"))
mpxmax = mmread(join(DATA_DIR, self.case_name, "opf", "xmax_AC.mtx"))
self.assertTrue(mfeq1(xmin, mpxmin.flatten()), msg)
self.assertTrue(mfeq1(xmax, mpxmax.flatten()), msg)
def test_constraints(self):
""" Test equality and inequality constraints.
"""
AA, ll, uu = self.solver._linear_constraints(self.om)
mpA = mmread(join(DATA_DIR, self.case_name, "opf", "A_DC.mtx"))
mpl = mmread(join(DATA_DIR, self.case_name, "opf", "l_DC.mtx"))
mpu = mmread(join(DATA_DIR, self.case_name, "opf", "u_DC.mtx"))
self.assertTrue(mfeq2(AA, mpA.tocsr()), self.case_name)
self.assertTrue(mfeq1(ll, mpl.flatten()), self.case_name)
self.assertTrue(mfeq1(uu, mpu.flatten()), self.case_name)
def test_var_bounds(self):
""" Test bounds on optimisation variables.
"""
_, xmin, xmax = self.solver._var_bounds()
# mpx0 = mmread(join(DATA_DIR, self.case_name, "opf", "x0_DC.mtx"))
mpxmin = mmread(join(DATA_DIR, self.case_name, "opf", "xmin_DC.mtx"))
mpxmax = mmread(join(DATA_DIR, self.case_name, "opf", "xmax_DC.mtx"))
# self.assertTrue(alltrue(x0 == mpx0.flatten()), self.case_name)
self.assertTrue(mfeq1(xmin, mpxmin.flatten()), self.case_name)
self.assertTrue(mfeq1(xmax, mpxmax.flatten()), self.case_name)
def testYbus(self):
""" Test bus and branch admittance matrices.
"""
self.case.index_buses()
Ybus, Yf, Yt = self.case.Y
mpYbus = mmread(join(DATA_DIR, self.case_name, "Ybus.mtx")).tocsr()
mpYf = mmread(join(DATA_DIR, self.case_name, "Yf.mtx")).tocsr()
mpYt = mmread(join(DATA_DIR, self.case_name, "Yt.mtx")).tocsr()
self.assertTrue(mfeq2(Ybus, mpYbus, diff=1e-12), self.case_name)
self.assertTrue(mfeq2(Yf, mpYf, diff=1e-12), self.case_name)
self.assertTrue(mfeq2(Yt, mpYt, diff=1e-12), self.case_name)
def testAy(self):
""" Test basin constraints for piece-wise linear gen cost variables.
"""
msg = self.case_name
self.case.sort_generators() # ext2int
_, _, gn = self.opf._remove_isolated(self.case)
_, ycon = self.opf._pwl_gen_costs(gn, self.case.base_mva)
if ycon is not None:
Ay = mmread(join(DATA_DIR, self.case_name, "opf", "Ay_DC.mtx"))
by = mmread(join(DATA_DIR, self.case_name, "opf", "by_DC.mtx"))
self.assertTrue(mfeq2(ycon.A, Ay.tocsr()), msg)
self.assertTrue(mfeq1(ycon.u, by.flatten()), msg)
def test_dSbus_dV(self):
""" Test partial derivative of power injection w.r.t. voltage.
"""
mpYbus = mmread(join(DATA_DIR, self.case_name, "Ybus.mtx")).tocsr()
mpV0 = mmread(join(DATA_DIR, self.case_name, "V0.mtx")).flatten()
dSbus_dVm, dSbus_dVa = self.case.dSbus_dV(mpYbus, mpV0)
mp_dSbus_dVm = mmread(join(DATA_DIR, self.case_name, "dSbus_dVm0.mtx"))
mp_dSbus_dVa = mmread(join(DATA_DIR, self.case_name, "dSbus_dVa0.mtx"))
self.assertTrue(mfeq2(dSbus_dVm, mp_dSbus_dVm.tocsr(), 1e-12),
self.case_name)
self.assertTrue(mfeq2(dSbus_dVa, mp_dSbus_dVa.tocsr(), 1e-12),
self.case_name)
def testVa(self):
""" Test voltage angle variable.
"""
msg = self.case_name
bs, _, _ = self.opf._remove_isolated(self.case)
_, refs = self.opf._ref_check(self.case)
Va = self.opf._get_voltage_angle_var(refs, bs)
mpVa0 = mmread(join(DATA_DIR, self.case_name, "opf", "Va0.mtx"))
mpVal = mmread(join(DATA_DIR, self.case_name, "opf", "Val.mtx"))
mpVau = mmread(join(DATA_DIR, self.case_name, "opf", "Vau.mtx"))
self.assertTrue(mfeq1(Va.v0, mpVa0.flatten()), msg)
self.assertTrue(mfeq1(Va.vl, mpVal.flatten()), msg)
self.assertTrue(mfeq1(Va.vu, mpVau.flatten()), msg)
def testPg(self):
""" Test active power variable.
"""
msg = self.case_name
self.case.sort_generators() # ext2int
_, _, gn = self.opf._remove_isolated(self.case)
Pg = self.opf._get_pgen_var(gn, self.case.base_mva)
mpPg0 = mmread(join(DATA_DIR, self.case_name, "opf", "Pg0.mtx"))
mpPmin = mmread(join(DATA_DIR, self.case_name, "opf", "Pmin.mtx"))
mpPmax = mmread(join(DATA_DIR, self.case_name, "opf", "Pmax.mtx"))
self.assertTrue(mfeq1(Pg.v0, mpPg0.flatten()), msg)
self.assertTrue(mfeq1(Pg.vl, mpPmin.flatten()), msg)
self.assertTrue(mfeq1(Pg.vu, mpPmax.flatten()), msg)
def testPfPt(self):
""" Test branch flow limit constraints.
"""
msg = self.case_name
_, ln, _ = self.opf._remove_isolated(self.case)
_, Bf, _, Pfinj = self.case.Bdc
Pf, Pt = self.opf._branch_flow_dc(ln, Bf, Pfinj, self.case.base_mva)
lpf = mmread(join(DATA_DIR, self.case_name, "opf", "lpf.mtx"))
upf = mmread(join(DATA_DIR, self.case_name, "opf", "upf.mtx"))
upt = mmread(join(DATA_DIR, self.case_name, "opf", "upt.mtx"))
self.assertTrue(mfeq1(Pf.l, lpf.flatten()), msg)
self.assertTrue(mfeq1(Pf.u, upf.flatten()), msg)
self.assertTrue(mfeq1(Pt.l, lpf.flatten()), msg)
self.assertTrue(mfeq1(Pt.u, upt.flatten()), msg)
def testPmis(self):
""" Test active power mismatch constraints.
"""
msg = self.case_name
case = self.case
case.sort_generators() # ext2int
B, _, Pbusinj, _ = case.Bdc
bs, _, gn = self.opf._remove_isolated(case)
Pmis = self.opf._power_mismatch_dc(bs, gn, B, Pbusinj, case.base_mva)
mpAmis = mmread(join(DATA_DIR, self.case_name, "opf", "Amis.mtx"))
mpbmis = mmread(join(DATA_DIR, self.case_name, "opf", "bmis.mtx"))
self.assertTrue(mfeq2(Pmis.A, mpAmis.tocsr(), 1e-12), msg)
self.assertTrue(mfeq1(Pmis.l, mpbmis.flatten()), msg)
self.assertTrue(mfeq1(Pmis.u, mpbmis.flatten()), msg)
def testVLConstPF(self):
""" Test dispatchable load, constant power factor constraints.
"""
msg = self.case_name
_, _, gn = self.opf._remove_isolated(self.case)
vl = self.opf._const_pf_constraints(gn, self.case.base_mva)
if vl.A.shape[0] != 0:
Avl = mmread(join(DATA_DIR, self.case_name, "opf", "Avl.mtx"))
self.assertTrue(mfeq2(vl.A, Avl.tocsr()), msg)
lvl = mmread(join(DATA_DIR, self.case_name, "opf", "lang.mtx"))
self.assertTrue(mfeq1(vl.l, lvl.flatten()), msg)
uvl = mmread(join(DATA_DIR, self.case_name, "opf", "uang.mtx"))
self.assertTrue(mfeq1(vl.u, uvl.flatten()), msg)
def test_simple_complex(self):
a = [[1, 2], [3, 4j]]
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (2, 2, 4, "array", "complex", "general"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def check_read(self, example, a, info):
f = open(self.fn, 'w')
f.write(example)
f.close()
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_simple_rectangular(self):
a = [[1,2,3],[4,5,6]]
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,3,6,'array','integer','general'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_real(self):
a = [[1,2],[3,4.0]]
fn = self.fn
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,2,4,'array','real','general'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_rectangular_real(self):
a = [[1, 2], [3.5, 4], [5, 6]]
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (3, 2, 6, "array", "real", "general"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_simple_skew_symmetric_float(self):
a = array([[1, 2], [-2.0, 4]], "f")
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (2, 2, 4, "array", "real", "skew-symmetric"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_simple_skew_symmetric(self):
a = [[1, 2], [-2, 4]]
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (2, 2, 4, "array", "integer", "skew-symmetric"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_simple_skew_symmetric(self):
a = [[1,2],[-2,4]]
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,2,4,'array','integer','skew-symmetric'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_complex(self):
a = [[1,2],[3,4j]]
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,2,4,'array','complex','general'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_skew_symmetric_float(self):
a = array([[1,2],[-2.0,4]],'f')
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,2,4,'array','real','skew-symmetric'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_hermitian(self):
a = [[1, 2 + 3j], [2 - 3j, 4]]
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (2, 2, 4, "array", "complex", "hermitian"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_simple_hermitian(self):
a = [[1,2+3j],[2-3j,4]]
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(2,2,4,'array','complex','hermitian'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_rectangular_real(self):
a = [[1,2],[3.5,4],[5,6]]
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(3,2,6,'array','real','general'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_simple_rectangular(self):
a = [[1, 2, 3], [4, 5, 6]]
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (2, 3, 6, "array", "integer", "general"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_random_rect_real(self):
sz = (20, 15)
a = rand(*sz)
fn = self.fn
mmwrite(fn, a)
assert_equal(mminfo(fn), (20, 15, 300, "array", "real", "general"))
b = mmread(fn)
assert_array_almost_equal(a, b)
def test_random_rect_real(self):
sz = (20,15)
a = rand(*sz)
fn = mktemp()
mmwrite(fn,a)
assert_equal(mminfo(fn),(20,15,300,'array','real','general'))
b = mmread(fn)
assert_array_almost_equal(a,b)
def test_integrate_solution(self):
""" Test integration of AC OPF solution.
"""
self.solver.solve()
bus = mmread(join(DATA_DIR, self.case_name, "opf", "Bus_AC.mtx"))
gen = mmread(join(DATA_DIR, self.case_name, "opf", "Gen_AC.mtx"))
branch = mmread(join(DATA_DIR, self.case_name, "opf", "Branch_AC.mtx"))
# FIXME: Improve accuracy.
pl = 4
# bus_i type Pd Qd Gs Bs area Vm Va baseKV zone Vmax Vmin lam_P lam_Q mu_Vmax mu_Vmin
for i, bs in enumerate(self.case.buses):
self.assertAlmostEqual(bs.v_magnitude, bus[i, 7], pl) # Vm
self.assertAlmostEqual(bs.v_angle, bus[i, 8], pl) # Va
self.assertAlmostEqual(bs.p_lmbda, bus[i, 13], pl) # lam_P
self.assertAlmostEqual(bs.q_lmbda, bus[i, 14], pl) # lam_Q
# FIXME: Improve accuracy
self.assertAlmostEqual(bs.mu_vmax, bus[i, 15], pl) # mu_Vmax
self.assertAlmostEqual(bs.mu_vmin, bus[i, 16], pl) # mu_Vmin
# bus Pg Qg Qmax Qmin Vg mBase status Pmax Pmin Pc1 Pc2 Qc1min Qc1max
# Qc2min Qc2max ramp_agc ramp_10 ramp_30 ramp_q apf mu_Pmax mu_Pmin
# mu_Qmax mu_Qmin
for i, gn in enumerate(self.case.generators):
# FIXME: Improve accuracy
self.assertAlmostEqual(gn.p, gen[i, 1], pl) # Pg
self.assertAlmostEqual(gn.q, gen[i, 2], pl) # Qg
self.assertAlmostEqual(gn.v_magnitude, gen[i, 5], pl) # Vg
self.assertAlmostEqual(gn.mu_pmax, gen[i, 21], pl) # mu_Pmax
self.assertAlmostEqual(gn.mu_pmin, gen[i, 22], pl) # mu_Pmin
self.assertAlmostEqual(gn.mu_qmax, gen[i, 23], pl) # mu_Qmax
self.assertAlmostEqual(gn.mu_qmin, gen[i, 24], pl) # mu_Qmin
# fbus tbus r x b rateA rateB rateC ratio angle status angmin angmax
# Pf Qf Pt Qt mu_Sf mu_St mu_angmin mu_angmax
for i, ln in enumerate(self.case.branches):
self.assertAlmostEqual(ln.p_from, branch[i, 13], pl) # Pf
self.assertAlmostEqual(ln.q_from, branch[i, 14], pl) # Qf
self.assertAlmostEqual(ln.p_to, branch[i, 15], pl) # Pt
self.assertAlmostEqual(ln.q_to, branch[i, 16], pl) # Qt
self.assertAlmostEqual(ln.mu_s_from, branch[i, 17], pl) # mu_Sf
self.assertAlmostEqual(ln.mu_s_to, branch[i, 18], pl) # mu_St
self.assertAlmostEqual(ln.mu_angmin, branch[i, 19], pl)
self.assertAlmostEqual(ln.mu_angmax, branch[i, 20], pl)
def testNewtonV(self):
""" Test the voltage vector solution from Newton's method.
"""
solution = NewtonPF(self.case).solve()
mpV = mmread(join(DATA_DIR, self.case_name, "V_Newton.mtx")).flatten()
self.assertTrue(mfeq1(solution["V"], mpV), self.case_name)
def test_simple_pattern(self):
a = scipy.sparse.csr_matrix([[0, 1.5], [3.0, 2.5]])
p = np.zeros_like(a.todense())
p[a.todense() > 0] = 1
info = (2, 2, 3, 'coordinate', 'pattern', 'general')
mmwrite(self.fn, a, field='pattern')
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_array_almost_equal(p, b.todense())
def test_empty_write_read(self):
# http://projects.scipy.org/scipy/ticket/883
b = scipy.sparse.coo_matrix((10, 10))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(10, 10, 0, 'coordinate', 'real', 'symmetric'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_initial_point(self):
""" Test selection of an initial interior point.
"""
b, l, g, _ = self.solver._unpack_model(self.om)
_, LB, UB = self.solver._var_bounds()
_, _, _, _, _, ny, _ = self.solver._dimension_data(b, l, g)
x0 = self.solver._initial_interior_point(b, g, LB, UB, ny)
mpx0 = mmread(join(DATA_DIR, self.case_name, "opf", "x0_AC.mtx"))
self.assertTrue(mfeq1(x0, mpx0.flatten()), self.case_name)
def check_read(self, example, a, info, dense, over32, over64):
with open(self.fn, 'w') as f:
f.write(example)
assert_equal(mminfo(self.fn), info)
if (over32 and (np.intp(0).itemsize < 8)) or over64:
assert_raises(OverflowError, mmread, self.fn)
else:
b = mmread(self.fn)
if not dense:
b = b.todense()
assert_equal(a, b)
def test_poly_costs(self):
""" Test quadratic costs.
"""
msg = self.case_name
base_mva = self.om.case.base_mva
b, l, g, _ = self.solver._unpack_model(self.om)
ipol, _, _, _, _, _, nxyz = self.solver._dimension_data(b, l, g)
Npol, Hpol, Cpol, fparm_pol, _, _ = \
self.solver._quadratic_costs(g, ipol, nxyz, base_mva)
if Npol is not None:
mpNpol = mmread(join(DATA_DIR, self.case_name, "opf", "Npol.mtx"))
mpHpol = mmread(join(DATA_DIR, self.case_name, "opf", "Hpol.mtx"))
mpCpol = mmread(join(DATA_DIR, self.case_name, "opf", "Cpol.mtx"))
mpfparm = mmread(join(DATA_DIR, self.case_name, "opf","fparm_pol.mtx"))
self.assertTrue(mfeq2(Npol, mpNpol.tocsr()), msg)
self.assertTrue(mfeq2(Hpol, mpHpol.tocsr()), msg)
self.assertTrue(mfeq1(Cpol, mpCpol.flatten()), msg)
self.assertTrue(mfeq2(fparm_pol, mpfparm), msg)
def test_read_geneal(self):
"""read a general matrix"""
fn = mktemp()
f = open(fn, 'w')
f.write(_general_example)
f.close()
assert_equal(mminfo(fn), (5, 5, 8, 'coordinate', 'real', 'general'))
a = [[1, 0, 0, 6, 0], [0, 10.5, 0, 0, 0], [0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 33.32], [0, 0, 0, 0, 12]]
b = mmread(fn).todense()
assert_array_almost_equal(a, b)
def test_read_symmetric(self):
"""read a symmetric matrix"""
fn = mktemp()
f = open(fn, 'w')
f.write(_symmetric_example)
f.close()
assert_equal(mminfo(fn), (5, 5, 7, 'coordinate', 'real', 'symmetric'))
a = [[1, 0, 0, 0, 0], [0, 10.5, 0, 250.5, 0], [0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 8], [0, 0, 0, 8, 12]]
b = mmread(fn).todense()
assert_array_almost_equal(a, b)
def test_read_symmetric_pattern(self):
fn = mktemp()
f = open(fn, 'w')
f.write(_symmetric_pattern_example)
f.close()
assert_equal(mminfo(fn),
(5, 5, 7, 'coordinate', 'pattern', 'symmetric'))
a = [[1, 0, 0, 0, 0], [0, 1, 0, 1, 0], [0, 0, 1, 0, 0],
[0, 1, 0, 1, 1], [0, 0, 0, 1, 1]]
b = mmread(fn).todense()
assert_array_almost_equal(a, b)
def test_empty_write_read(self):
# https://github.com/scipy/scipy/issues/1410 (Trac #883)
b = scipy.sparse.coo_matrix((10, 10))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(10, 10, 0, 'coordinate', 'real', 'symmetric'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_integrate_solution(self):
""" Test integration of AC OPF solution.
"""
self.solver.solve()
bus = mmread(join(DATA_DIR, self.case_name, "opf", "Bus_AC.mtx"))
gen = mmread(join(DATA_DIR, self.case_name, "opf", "Gen_AC.mtx"))
branch = mmread(join(DATA_DIR, self.case_name, "opf", "Branch_AC.mtx"))
# FIXME: Improve accuracy.
pl = 4
# bus_i type Pd Qd Gs Bs area Vm Va baseKV zone Vmax Vmin lam_P lam_Q mu_Vmax mu_Vmin
for i, bs in enumerate(self.case.buses):
self.assertAlmostEqual(bs.v_magnitude, bus[i, 7], pl) # Vm
self.assertAlmostEqual(bs.v_angle, bus[i, 8], pl) # Va
self.assertAlmostEqual(bs.p_lmbda, bus[i, 13], pl) # lam_P
self.assertAlmostEqual(bs.q_lmbda, bus[i, 14], pl) # lam_Q
# FIXME: Improve accuracy
self.assertAlmostEqual(bs.mu_vmax, bus[i, 15], pl) # mu_Vmax
self.assertAlmostEqual(bs.mu_vmin, bus[i, 16], pl) # mu_Vmin
def testV0(self):
""" Test the initial voltage vector.
"""
solver = _ACPF(self.case)
b, _, g, _, _, _, _ = solver._unpack_case(self.case)
self.case.index_buses(b)
V0 = solver._initial_voltage(b, g)
mpV0 = mmread(join(DATA_DIR, self.case_name, "V0.mtx")).flatten()
self.assertTrue(mfeq1(V0, mpV0), self.case_name)
def test_read_hermitian(self):
fn = self.fn
f = open(fn, 'w')
f.write(_hermitian_example)
f.close()
assert_equal(mminfo(fn),
(5, 5, 7, 'coordinate', 'complex', 'hermitian'))
a = [[1, 0, 0, 0, 0], [0, 10.5, 0, 250.5 - 22.22j, 0],
[0, 0, .015, 0, 0], [0, 250.5 + 22.22j, 0, -280, -33.32j],
[0, 0, 0, 33.32j, 12]]
b = mmread(fn).todense()
assert_array_almost_equal(a, b)
def test_solution(self):
""" Test AC OPF solution.
"""
msg = self.case_name
solution = self.solver.solve()
lmbda = solution["lmbda"]
f = mmread(join(DATA_DIR, self.case_name, "opf", "f_AC.mtx"))
x = mmread(join(DATA_DIR, self.case_name, "opf", "x_AC.mtx"))
diff = 1e-4
# FIXME: Improve accuracy.
self.assertAlmostEqual(solution["f"], f[0], places=3)
self.assertTrue(mfeq1(solution["x"], x.flatten(), diff))
if len(lmbda["mu_l"]) > 0:
mu_l = mmread(join(DATA_DIR, self.case_name, "opf", "mu_l_AC.mtx"))
self.assertTrue(mfeq1(lmbda["mu_l"], mu_l.flatten(), diff), msg)
if len(lmbda["mu_u"]) > 0:
mu_u = mmread(join(DATA_DIR, self.case_name, "opf", "mu_u_AC.mtx"))
self.assertTrue(mfeq1(lmbda["mu_u"], mu_u.flatten(), diff), msg)
if len(lmbda["lower"]) > 0:
muLB = mmread(join(DATA_DIR, self.case_name, "opf", "muLB_AC.mtx"))
# FIXME: Improve accuracy.
self.assertTrue(mfeq1(lmbda["lower"], muLB.flatten(), diff), msg)
if len(lmbda["upper"]) > 0:
muUB = mmread(join(DATA_DIR, self.case_name, "opf", "muUB_AC.mtx"))
# FIXME: Improve accuracy.
self.assertTrue(mfeq1(lmbda["upper"], muUB.flatten(), diff), msg)
def test_real_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'real', 'general'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_complex_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'complex', 'general'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_coefficient_transformation(self):
""" Test transformation of quadratic coefficients for w into
coefficients for X.
"""
msg = self.case_name
base_mva = self.om.case.base_mva
b, l, g, _ = self.solver._unpack_model(self.om)
ipol, ipwl, _, _, nw, ny, nxyz = self.solver._dimension_data(b, l, g)
Npwl, Hpwl, Cpwl, fparm_pwl, any_pwl = \
self.solver._pwl_costs(ny, nxyz, ipwl)
Npol, Hpol, Cpol, fparm_pol, polycf, npol = \
self.solver._quadratic_costs(g, ipol, nxyz, base_mva)
NN, HHw, CCw, ffparm = \
self.solver._combine_costs(Npwl, Hpwl, Cpwl, fparm_pwl, any_pwl,
Npol, Hpol, Cpol, fparm_pol, npol, nw)
HH, CC, _ = \
self.solver._transform_coefficients(NN, HHw, CCw, ffparm, polycf,
any_pwl, npol, nw)
mpHH = mmread(join(DATA_DIR, self.case_name, "opf", "HH.mtx"))
mpCC = mmread(join(DATA_DIR, self.case_name, "opf", "CC.mtx"))
self.assertTrue(mfeq2(HH, mpHH.tocsr()), msg)
self.assertTrue(mfeq1(CC, mpCC.flatten()), msg)
def test_combine_costs(self):
""" Test combination of pwl and poly costs.
"""
msg = self.case_name
base_mva = self.om.case.base_mva
b, l, g, _ = self.solver._unpack_model(self.om)
ipol, ipwl, _, _, nw, ny, nxyz = self.solver._dimension_data(b, l, g)
Npwl, Hpwl, Cpwl, fparm_pwl, any_pwl = self.solver._pwl_costs(ny, nxyz,
ipwl)
Npol, Hpol, Cpol, fparm_pol, _, npol = \
self.solver._quadratic_costs(g, ipol, nxyz, base_mva)
NN, HHw, CCw, ffparm = \
self.solver._combine_costs(Npwl, Hpwl, Cpwl, fparm_pwl, any_pwl,
Npol, Hpol, Cpol, fparm_pol, npol, nw)
mpNN = mmread(join(DATA_DIR, self.case_name, "opf", "NN.mtx"))
mpHHw = mmread(join(DATA_DIR, self.case_name, "opf", "HHw.mtx"))
mpCCw = mmread(join(DATA_DIR, self.case_name, "opf", "CCw.mtx"))
mpffparm = mmread(join(DATA_DIR, self.case_name, "opf", "ffparm.mtx"))
self.assertTrue(mfeq2(NN, mpNN.tocsr()), msg)
self.assertTrue(mfeq2(HHw, mpHHw.tocsr()), msg)
self.assertTrue(mfeq1(CCw, mpCCw.flatten()), msg)
self.assertTrue(mfeq2(ffparm, mpffparm), msg)
def _mmio_example(m=100, n=100, k=10):
print '\nTesting mmio read and write ...\n'
import scipy.io.mmio as mmio
W_org = random.rand(m, k)
H_org = random.rand(n, k)
X = W_org.dot(H_org.T)
X[random.rand(n, k) < 0.5] = 0
X_sparse = sps.csr_matrix(X)
filename = '_temp_mmio.mtx'
mmio.mmwrite(filename, X_sparse)
A = mmio.mmread(filename)
alg = NMF_ANLS_BLOCKPIVOT()
rslt = alg.run(X_sparse, k, max_iter=50)
def createInput(input_file, a_file, x_file, q, N) : #print 'reading the matrix' a = mmread(input_file) #print 'done' afile = open(a_file, 'w+') xfile = open(x_file, 'w+') elements = zip(a.row, a.col, a.data) n = len(elements) p = max(a.row) #generate the vector x x = [] #n = int(sys.argv[2]) #elemRange = int(sys.argv[3]) for i in range(0, q) : x.append(random.uniform(-1, 1)) print >>afile, n, p, q, N #a for i,j,v in elements : print >> afile, v, print >> afile, '' #s s = random.sample(range(n), p) s.sort() print >>afile, 0, for i in range(1, p-1) : print >> afile, s[i], print >> afile, n #x for xx in x: print >>xfile, xx, print >> xfile, '' #k #k = random.sample(range(q), n) #k.sort() for i in range(0, n) : print >> afile, int(random.uniform(0, q)), print >> afile, ''
def createTrainingData():
finalAttributeList = []
inputDir = "./sparseMatrixdataSet/"
for fileName in tqdm(os.listdir(inputDir)):
fileNameWithPath = inputDir + fileName
matData = mmread(fileNameWithPath)
if ((sp.isspmatrix_coo(matData) == True)
or ((sp.isspmatrix_coo(matData) == False) and
(matData.shape[1] != 1))):
attributeList = calAttributes(matData)
finalAttributeList.append(attributeList)
else:
print(fileName + " is not a sparse matrix.\n")
finalArr = numpy.asarray(finalAttributeList)
numpy.savetxt("spmvData.csv", finalArr, fmt='%f', delimiter=",")
print("Training data is stored in spmvData.csv\n")
def multiply(fname):
mat = mmread(fname)
rows, cols, nnzs = mminfo(fname)[0:3]
vec = np.ones((rows, 1), dtype=np.int)
matcoo = mat.tocoo()
start1 = timeit.default_timer()
matcoo.dot(vec)
stop1 = timeit.default_timer()
t1 = (stop1 - start1) * math.pow(10, 6)
matcsr = mat.tocsr()
start2 = timeit.default_timer()
matcsr.dot(vec)
stop2 = timeit.default_timer()
t2 = (stop2 - start2) * math.pow(10, 6)
matcsc = mat.tocsc()
start3 = timeit.default_timer()
matcsc.dot(vec)
stop3 = timeit.default_timer()
t3 = (stop3 - start3) * math.pow(10, 6)
matlil = mat.tolil()
start4 = timeit.default_timer()
matlil.dot(vec)
stop4 = timeit.default_timer()
t4 = (stop4 - start4) * math.pow(10, 6)
matdia = mat.todia()
start5 = timeit.default_timer()
matdia.dot(vec)
stop5 = timeit.default_timer()
t5 = (stop5 - start5) * math.pow(10, 6)
matdok = mat.todok()
start6 = timeit.default_timer()
matdok.dot(vec)
stop6 = timeit.default_timer()
t6 = (stop6 - start6) * math.pow(10, 6)
return rows, cols, nnzs, t1, t2, t3, t4, t5, t6
def test_sparse_formats(self):
mats = []
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
mats.append(scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5)))
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
mats.append(scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5)))
for mat in mats:
expected = mat.todense()
for fmt in ['csr', 'csc', 'coo']:
fn = mktemp(dir=self.tmpdir) # safe, we own tmpdir
mmwrite(fn, mat.asformat(fmt))
result = mmread(fn).todense()
assert_array_almost_equal(result, expected)
def get_data_from_10x(input_dir):
matrix_file = os.path.join(input_dir, "matrix.mtx")
barcodes_file = os.path.join(input_dir, "barcodes.tsv")
peaks_file = os.path.join(input_dir, "peaks.bed")
# use sparse matrix to save memory
data = mmread(matrix_file).astype(dtype=np.int8)
data = sp_sparse.csr_matrix(data)
barcodes = list()
peaks = list()
with open(barcodes_file) as f:
for line in f:
ll = line.strip().split("\t")
barcodes.append(ll[0])
with open(peaks_file) as f:
for line in f:
ll = line.strip().split("\t")
peaks.append(ll[0] + ":" + ll[1] + "-" + ll[2])
return data, barcodes, peaks
def test_gzip_py3(self):
# test if fix for #2152 works
try:
# gzip module can be missing from Python installation
import gzip
except:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_gzip = "%s.gz" % self.fn
with open(self.fn, 'rb') as f_in:
f_out = gzip.open(fn_gzip, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_gzip).todense()
assert_array_almost_equal(a, b.todense())
def test_bzip2_py3(self):
# test if fix for #2152 works
try:
# bz2 module isn't always built when building Python.
import bz2
except:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_bzip2 = "%s.bz2" % self.fn
with open(self.fn, 'rb') as f_in:
f_out = bz2.BZ2File(fn_bzip2, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_bzip2).todense()
assert_array_almost_equal(a, b.todense())
def read_matrix(self, filename):
"""
reads grapchi format in the numpy array. The order is wrong in grapchi, therefore, we play tricks.
About the second (order of rows vs. columns), we have a problem that in distributed graphlab we
output the matrix by rows, since each node is a row, and nodes are on different machines.
So to be compatible I left GraphChi code to confirm to GraphLab. In case it helps, here is a matlab mmread.m
function which switches the order to be sorted by rows http://select.cs.cmu.edu/code/graphlab/mmread.m
I am not sure if it is easy to fix it in python or not. [...]
You are definitely right the the standard describes column
order and not row order.
"""
logger.debug("Loading matrix market matrix ")
f = open(filename, "rb")
r = mmread(f)
m = np.array(r.ravel())
m = m.reshape(r.shape, order='F')
# print "mmread", r.ravel(), r.shape
# print "trans", m.ravel(), m.shape
# if m.shape[0] > 1:
# print m[1]
f.close()
return m
def generate_mm_and_plot(matrixname):
matrixName=matrixname+ ".mtx"
mm = open(matrixname +'.mm', 'w')
outputName=matrixname + ".png"
#outputName=matrixname + ".pdf"
(n,n,nz,void,fld,symm) = mminfo(matrixName)
A = mmread(matrixName).tocsr().tocoo()
# creating a file in coo format
#print n, n, A.size
mm.write( str(n) + ' ' + str(n) + ' ' + str(A.size) + '\n')
for i in range(A.size):
mm.write( str(A.row[i])+ ' ' + str(A.col[i])+ ' ' + str(A.data[i])+ '\n')
#print A.row[i]+1, A.col[i]+1 , A.data[i]
# end for
#print repr(A)
#print A.col
#print A.size
# end of creating a file in coo format
#pylab.title("Matrix :" + matrixname, fontsize=22)
pylab.title("Matrix: " + matrixname + ", n:" + str(n) + ", nz:" + str(A.size) + '\n' ,fontsize=10 )
pylab.spy(A,marker='.',markersize=1)
pylab.savefig(outputName,format=None)
def test_solution(self):
""" Test DC OPF solution.
"""
msg = self.case_name
solution = self.solver.solve()
lmbda = solution["lmbda"]
mpf = mmread(join(DATA_DIR, self.case_name, "opf", "f_DC.mtx"))
mpx = mmread(join(DATA_DIR, self.case_name, "opf", "x_DC.mtx"))
mpmu_l = mmread(join(DATA_DIR, self.case_name, "opf", "mu_l_DC.mtx"))
mpmu_u = mmread(join(DATA_DIR, self.case_name, "opf", "mu_u_DC.mtx"))
mpmuLB = mmread(join(DATA_DIR, self.case_name, "opf", "muLB_DC.mtx"))
mpmuUB = mmread(join(DATA_DIR, self.case_name, "opf", "muUB_DC.mtx"))
diff = 1e-09
self.assertAlmostEqual(solution["f"], mpf[0], places=6)
self.assertTrue(mfeq1(solution["x"], mpx.flatten(), diff), msg)
self.assertTrue(mfeq1(lmbda["mu_l"], mpmu_l.flatten(), diff), msg)
self.assertTrue(mfeq1(lmbda["mu_u"], mpmu_u.flatten(), diff), msg)
self.assertTrue(mfeq1(lmbda["lower"], mpmuLB.flatten(), diff), msg)
self.assertTrue(mfeq1(lmbda["upper"], mpmuUB.flatten(), diff), msg)
def plot(self, matrix, RHSvector, log='auto'):
import tempfile
import os
if "print" in os.environ['FIPY_DISPLAY_MATRIX'].lower().split():
print("-" * 75)
print(self.title)
print("-" * 75)
print("L:")
print(matrix)
print("b:", RHSvector)
(f, mtxName) = tempfile.mkstemp(suffix='.mtx')
matrix.exportMmf(mtxName)
mtx = mmio.mmread(mtxName)
os.remove(mtxName)
pyplot.ion()
c = mtx.tocoo()
y = c.row
x = c.col
z = c.data
N = matrix._shape[0]
b = RHSvector
if numerix.shape(b) == ():
b = numerix.zeros((N, ), 'l')
if len(z) == 0:
y = numerix.zeros((1, ), 'l')
x = numerix.zeros((1, ), 'l')
z = numerix.zeros((1, ), 'l')
def signed_to_logs(v):
return (numerix.where(v > 0, numerix.log10(v), numerix.nan),
numerix.where(v < 0, numerix.log10(-v), numerix.nan))
def logs_to_signed(v, plus, minus):
v = numerix.where(v > 0, plus, -minus)
v = numerix.where(numerix.isnan(v), 0., v)
return v
zPlus, zMinus = signed_to_logs(z)
bPlus, bMinus = signed_to_logs(b)
logs = (zPlus, zMinus, bPlus, bMinus)
log = ((log == True)
or (log == 'auto' and
(numerix.nanmax(numerix.concatenate(logs)) -
numerix.nanmin(numerix.concatenate(logs)) > 2)))
if log:
zMin = numerix.nanmin(numerix.concatenate(logs))
zMax = numerix.nanmax(numerix.concatenate(logs))
zMin -= 0.5
numdec = numerix.floor(zMax) - numerix.ceil(zMin)
if numdec < 0:
zMax += 0.5
for v in logs:
v -= zMin
zRange = zMax - zMin
if zRange == 0:
zRange = numerix.nanmax(zPlus) + 1
z = logs_to_signed(z, zPlus, zMinus)
b = logs_to_signed(b, bPlus, bMinus)
fmt = SignedLogFormatter(threshold=zMin)
loc = SignedLogLocator(threshold=zMin)
else:
zRange = max(abs(numerix.concatenate((z, b))))
if zRange == 0:
zRange = 1
fmt = None
loc = None
pyplot.ioff()
fig = pyplot.figure(self.id)
fig.clf()
usetex = rcParams['text.usetex']
rcParams['text.usetex'] = False
cmap = cm.RdBu
norm = Normalize(vmin=-zRange, vmax=zRange)
x0 = self.margin
L_ax = fig.add_axes([
x0 / self.aspect, self.margin, self.L_width / self.aspect,
self.L_width
])
L_ax.text(0.5,
-0.1,
"L",
transform=L_ax.transAxes,
horizontalalignment='center',
verticalalignment='baseline')
x0 += self.L_width + self.buffer
c_ax = fig.add_axes([
x0 / self.aspect, self.margin, self.c_width / self.aspect,
self.L_width
])
x0 += self.c_width + self.buffer
b_ax = fig.add_axes([
x0 / self.aspect, self.margin, self.b_width / self.aspect,
self.L_width
],
sharey=L_ax)
b_ax.text(0.5,
-0.1,
"b",
transform=b_ax.transAxes,
horizontalalignment='center',
verticalalignment='baseline')
def scatterRectangles(x, y, z, norm=None, cmap=None):
patches = [
Rectangle(numerix.array([X - 0.5, Y - 0.5]),
1.,
1.,
edgecolor='none') for X, Y in zip(x, y)
]
collection = PatchCollection(patches,
norm=norm,
cmap=cmap,
edgecolors='none')
collection.set_array(z)
return collection
L_ax.add_collection(
scatterRectangles(x=x, y=y, z=z, norm=norm, cmap=cmap))
b_ax.add_collection(
scatterRectangles(x=numerix.zeros((N, ), 'l'),
y=numerix.arange(N),
z=b,
norm=norm,
cmap=cmap))
ColorbarBase(ax=c_ax,
cmap=cmap,
norm=norm,
orientation='vertical',
format=fmt,
ticks=loc)
pyplot.setp((b_ax.get_xticklabels(), b_ax.get_yticklabels(),
b_ax.get_xticklines(), b_ax.get_yticklines()),
visible=False)
L_ax.set_xlim(xmin=-0.5, xmax=N - 0.5)
L_ax.set_ylim(ymax=-0.5, ymin=N - 0.5)
b_ax.set_xlim(xmin=-0.5, xmax=0.5)
b_ax.set_ylim(ymax=-0.5, ymin=N - 0.5)
fig.suptitle(self.title, x=0.5, y=0.95, fontsize=14)
pyplot.draw()
rcParams['text.usetex'] = usetex
dest='gamma',
type='float',
default=0.0001,
help='gradient ascent learning rate gamma (default: %default)')
parser.add_option('--max_iters',
dest='max_iters',
type='int',
default=25,
help='max iterations (default: %default)')
(opts, args) = parser.parse_args()
if not opts.train or not opts.D or not opts.lbda or not opts.gamma:
parser.print_help()
raise SystemExit
data = mmread(opts.train).tocsr(
) # this converts a 1-indexed file to a 0-indexed sparse array
if opts.test:
testdata = mmread(opts.test).tocsr()
U = 0.01 * np.random.random_sample((data.shape[0], opts.D))
V = 0.01 * np.random.random_sample((data.shape[1], opts.D))
num_train_sample_users = min(data.shape[0], 1000)
train_sample_users = random.sample(xrange(data.shape[0]),
num_train_sample_users)
print 'train mrr = {0:.4f}'.format(
compute_mrr(data, U, V, train_sample_users))
if opts.test:
num_test_sample_users = min(testdata.shape[0], 1000)
test_sample_users = random.sample(xrange(testdata.shape[0]),
num_test_sample_users)
import timeit
import inspect
import numpy as np
import scipy
from scipy import sparse
import matplotlib.pyplot as plt
from scipy.io import mmio
import os
print(os.getcwd())
#
#os.chdir('/home/shilu')
#print(os.getcwd())
#B162 = mmio.mmread("smdsurv162.mtx").tocsc()
A40520 = mmio.mmread('smdsurv40520.mtx').tocsc()
Atilde = mmio.mmread("modsurv40520.mtx").tocsc()
maxIterations = 300
iterSpace = np.linspace(0,maxIterations,num=maxIterations)
residualVector = np.array([])
sparseResidual= []
def sparseDirect (A):
b=np.ones(A.shape[0])
return scipy.sparse.linalg.spsolve(A,b)
def sparseError(A):
b=np.ones(A.shape[0])
return np.linalg.norm(A*sparseDirect(A)-b)/np.linalg.norm(b)