def gen_group(fnin, sub_json, strict=False, verbose=False):
print('Loading data')
Ads, bs = loadc_Ads_bs([fnin], ico=True)
print('Sub: %u rows' % len(Ads))
iold = instances(Ads)
names_old = index_names(Ads)
run_sub_json(Ads, sub_json, strict=strict, verbose=verbose)
names = index_names(Ads)
print("Sub: %u => %u names" % (len(names_old), len(names)))
print('Sub: %u => %u instances' % (iold, instances(Ads)))
for row_ds, row_bs in zip(Ads, bs):
yield row_ds, row_bs
def run(fns_in, sub_json=None, verbose=False):
assert len(fns_in) > 0
# arbitrary corner...data is thrown away
Ads, b = loadc_Ads_b(fns_in, "slow_max")
if sub_json:
print('Subbing JSON %u rows' % len(Ads))
#pds(Ads, 'Orig')
names_old = index_names(Ads)
run_sub_json(Ads, sub_json, verbose=verbose)
names_new = index_names(Ads)
print("Sub: %u => %u names" % (len(names_old), len(names_new)))
print(names_new)
print('Subbed JSON %u rows' % len(Ads))
names = names_new
#pds(Ads, 'Sub')
else:
names = index_names(Ads)
# Squash into a matrix
# A_ub2, b_ub2 = Adi2matrix_random(A_ubd, b, names)
Amat, _bmat = Ads2matrix_linear(Ads, b)
#pmatrix(Amat, 'Matrix')
'''
The matrix must be fully ranked to even be considered reasonable
Even then, floating point error *possibly* could make it fully ranked, although probably not since we have whole numbers
Hence the slogdet check
'''
print
# https://docs.scipy.org/doc/numpy-dev/reference/generated/numpy.linalg.matrix_rank.html
rank = np.linalg.matrix_rank(Amat)
print('rank: %s / %d col' % (rank, len(names)))
# doesn't work on non-square matrices
if 0:
# https://docs.scipy.org/doc/numpy-1.14.0/reference/generated/numpy.linalg.slogdet.html
sign, logdet = np.linalg.slogdet(Amat)
# If the determinant is zero, then sign will be 0 and logdet will be -Inf
if sign == 0 and logdet == float('-inf'):
print('slogdet :( : 0')
else:
print('slogdet :) : %s, %s' % (sign, logdet))
if rank != len(names):
raise Exception("Matrix not fully ranked w/ %u / %u" %
(rank, len(names)))
def __init__(self, Ads, drop_names=[]):
self.Ads = Ads
self.names = index_names(self.Ads)
# known zero delay elements
self.drop_names = set(drop_names)
# active names in rows
# includes sub variables, excludes variables that have been substituted out
self.base_names = set(self.names)
self.names = set(self.base_names)
# List of variable substitutions
# k => dict of v:n entries that it came from
self.subs = {}
self.verbose = True
def check_cols():
assert len(index_names(Ads)) == cols
def massage_equations(Ads, b, verbose=False, corner=None):
'''
Subtract equations from each other to generate additional constraints
Helps provide additional guidance to solver for realistic delays
Ex: given:
a >= 10
a + b >= 100
A valid solution is:
a = 100
However, a better solution is something like
a = 10
b = 90
This creates derived constraints to provide more realistic results
Equation pipeline
Some operations may generate new equations
Simplify after these to avoid unnecessary overhead on redundant constraints
Similarly some operations may eliminate equations, potentially eliminating a column (ie variable)
Remove these columns as necessary to speed up solving
'''
assert len(Ads) == len(b), 'Ads, b length mismatch'
def check_cols():
assert len(index_names(Ads)) == cols
def debug(what):
check_cols()
if 1 or verbose:
print('')
print_eqns(Ads, b, verbose=verbose, label=what, lim=20)
col_dist(Ads, what)
#check_feasible_d(Ads, b)
# Try to (intelligently) subtract equations to generate additional constraints
# This helps avoid putting all delay in a single shared variable
dstart = len(b)
cols = len(index_names(Ads))
# Each iteration one more column is allowed until all columns are included
# (and the system is stable)
col_lim = 15
di = 0
while True:
print
n_orig = len(b)
print('Loop %d, lim %d' % (di + 1, col_lim))
# Meat of the operation
Ads, b = derive_eq_by_row(Ads, b, col_lim=col_lim, cmp_heuristic=True)
debug("der_rows")
# Run another simplify pass since new equations may have overlap with original
Ads, b = simplify_rows(Ads, b, corner=corner)
print('Derive row: %d => %d equations' % (n_orig, len(b)))
debug("der_rows simp")
# derive_cols is mostly degenerate case of derive_rows
# however, it will sub out single variables a lot faster if there are a lot of them
# linear vs above quadratic, might as well keep it in
if 1:
n_orig2 = len(b)
# Meat of the operation
Ads, b = derive_eq_by_col(Ads, b)
debug("der_cols")
# Run another simplify pass since new equations may have overlap with original
Ads, b = simplify_rows(Ads, b, corner=corner)
print('Derive col %d: %d => %d equations' %
(di + 1, n_orig2, len(b)))
debug("der_cols simp")
# Doesn't help computation, but helps debugging
Ads, b = sort_equations(Ads, b)
debug("loop done")
col_dist(Ads, 'derive done iter %d, lim %d' % (di, col_lim), lim=12)
rows = len(Ads)
# possible that a new equation was generated and taken away, but close enough
if n_orig == len(b) and col_lim >= cols:
break
col_lim += col_lim / 5
di += 1
dend = len(b)
print('')
print('Derive net: %d => %d' % (dstart, dend))
print('')
# Was experimentting to see how much the higher order columns really help
# Helps debug readability
Ads, b = sort_equations(Ads, b)
debug("final (sorted)")
print('')
print('Massage final: %d => %d rows' % (dstart, dend))
cols_end = len(index_names(Ads))
print('Massage final: %d => %d cols' % (cols, cols_end))
assert cols_end == cols
return Ads, b
def run(fns_in,
corner,
run_corner,
sub_json=None,
bounds_csv=None,
dedup=True,
massage=False,
outfn=None,
verbose=False,
**kwargs):
print('Loading data')
Ads, b = loadc_Ads_b(fns_in, corner, ico=True)
# Remove duplicate rows
# is this necessary?
# maybe better to just add them into the matrix directly
if dedup:
oldn = len(Ads)
iold = instances(Ads)
Ads, b = simplify_rows(Ads, b, corner=corner)
print('Simplify %u => %u rows' % (oldn, len(Ads)))
print('Simplify %u => %u instances' % (iold, instances(Ads)))
if sub_json:
print('Sub: %u rows' % len(Ads))
iold = instances(Ads)
names_old = index_names(Ads)
run_sub_json(Ads, sub_json, verbose=verbose)
names = index_names(Ads)
print("Sub: %u => %u names" % (len(names_old), len(names)))
print('Sub: %u => %u instances' % (iold, instances(Ads)))
else:
names = index_names(Ads)
'''
Substitution .csv
Special .csv containing one variable per line
Used primarily for multiple optimization passes, such as different algorithms or additional constraints
'''
if bounds_csv:
Ads2, b2 = loadc_Ads_b([bounds_csv], corner, ico=True)
bounds = Ads2bounds(Ads2, b2)
assert len(bounds), 'Failed to load bounds'
rows_old = len(Ads)
Ads, b = filter_bounds(Ads, b, bounds, corner)
print('Filter bounds: %s => %s + %s rows' %
(rows_old, len(Ads), len(Ads2)))
Ads = Ads + Ads2
b = b + b2
assert len(Ads) or allow_zero_eqns()
assert len(Ads) == len(b), 'Ads, b length mismatch'
if verbose:
print
print_eqns(Ads, b, verbose=verbose)
#print
#col_dist(A_ubd, 'final', names)
if massage:
try:
Ads, b = massage_equations(Ads, b, corner=corner)
except SimplifiedToZero:
if not allow_zero_eqns():
raise
print('WARNING: simplified to zero equations')
Ads = []
b = []
print('Converting to numpy...')
names, Anp = A_ds2np(Ads)
run_corner(Anp,
np.asarray(b),
names,
corner,
outfn=outfn,
verbose=verbose,
**kwargs)
