def run(self, run_id):
# queue input into the write input slot,
# execute oprator if all inputs are available
if not self.ready(): return
newinputs = map(ArrayStore.instance().get, self.slots)
inputshapes = [arr.shape for arr in newinputs]
self.inputs[run_id] = list(self.slots)
self.slots = [None] * self.nargs
wlog.info('%s.run(%d)', self.op, run_id)
pstore = self.op.pstore(run_id)
start = time.time()
output, stats = self.op.run(newinputs, run_id)
runtime = time.time() - start
newinputs = None
# calculate runtime and provenance overheads
pstore.close()
if pstore.get_stat('write') > 0:
for f in ('outcache', 'incache', 'serin', 'serout', 'mergcost', 'bdbcost',
'keycost', 'idxcost', 'write', 'flush'):
wlog.debug( "%s\t%f", f, pstore.get_stat(f) )
# store outputs
outputid = ArrayStore.instance().add(output)
outputshape = output.shape
del output
self.outputs[run_id] = outputid
outputdisk = ArrayStore.instance().size(outputid)
Stats.instance().add_wrun(run_id, self.op, runtime, inputshapes, outputshape, outputdisk, pstore)
return outputid
"""
exit()
dbname = './_output/pablostats.db'
runmodes = []
for arg in sys.argv:
if '.db' in arg:
dbname = arg
else:
try:
runmode = int(arg)
runmodes.append(runmode)
except:
pass
stats = Stats.instance(dbname)
conn = stats.conn
cur = conn.cursor()
def get_exps(runmode):
# get all the experiments
cur.execute("""select rowid, runmode, runtype, width, height, diskconstraint, runconstraint
from exec where runmode = ? and
runtype not in ('noop', 'noop_model', 'stats', 'stats_model', 'opt', 'noop_m', 'stats_m')
order by rowid, diskconstraint""", (runmode,))
return cur.fetchall()
def get_labels(exps):
replaces = (('KEY', 'REF'), ('PTR_', '3_'), ('_B', '_b'), ('_F', '_f'), ('PTMAP_', '2_')
)
def run_nlp(stats, w, mp, maxdisk, maxoverhead):
"""
maxoverhead is percentage of average runtime cost of optimizable operators
"""
stats = Stats.instance()
ops = w.get_optimizable_ops()
mapops = [op for op in ops if Mode.FULL_MAPFUNC in op.supported_modes()]
ops = [op for op in ops if Mode.FULL_MAPFUNC not in op.supported_modes()]
matstrats = w.get_matstrats()
currun = w._runid
pairs = [(currun, op, s) for op in ops for s in matstrats]
trips = list(pairs)
existingops = []
for r,o,s in Runtime.instance().get_disk_strategies():
trips.append((r,o,s))
trips.append((r,o,Strat.query()))
existingops.append((o,r))
xold = [Runtime.instance().check_strategy(op,r,s) and 1 or 0 for r,op,s in trips]
avg_runtime = sum(filter(lambda x: x > 0, [mp.get_opcost(op, s) for r,op,s in pairs if r == currun]))
maxoverhead *= avg_runtime
G1 = []
G1dict = {}
for r,op,s in trips:
if op not in G1dict:
G1dict[op] = []
if r == currun:
disk = mp.get_disk(op, s)
else:
disk = stats.get_disk(r,op,s)
G1.append(disk)
if disk > 0:
G1dict[op].append(disk)
G2 = []
G2dict = {}
for r,op,s in trips:
if op not in G2dict:
G2dict[op] = []
if r == currun:
ov = mp.get_provcost(op, s)
else:
ov = 0.0
G2.append(ov)
if ov > 0:
G2dict[op].append(ov)
G = matrix([G1, G2]).trans()
h = matrix([maxdisk, maxoverhead])
A = []
# every operator needs exactly 1 strategy constraint
blocksize = len(matstrats)
for i in xrange(len(ops)):
row = [0.0] * len(trips)
for col in xrange(len(trips)):
if blocksize * i <= col and col < blocksize * (i+1) and col < len(pairs):
row[col] = 1.0
A.append(row)
for i, (op,r) in enumerate(existingops):
row = [0.0] * len(trips)
col = len(pairs) + i * 2
row[col] = 1.0
row[col+1] = 1.0
A.append(row)
A = matrix(A).trans()
b = matrix([1.0] * (len(ops)+len(existingops)))
c = []
mincs = {}
for r,op,s in trips:
cost = mp.get_pqcost(op,s,r)
c.append(cost)
if cost > 0 and op not in mincs:
mincs[op] = cost
if cost > 0 and cost < mincs[op]:
mincs[op] = cost
if 'CreateModel' in str(op) and cost > 10000:
import pdb
pdb.set_trace()
# normalize disk and runcost to minc
G1p, G2p = [], []
for (r,o,s), g1, g2 in zip(trips, G1, G2):
G1p.append(g1 / max(G1dict[o]) * mincs[o])
G2p.append(g2 / max(G2dict[o]) * mincs[o])
c = map(sum, zip(c, G1p, G2p))
cp = list(c)
d = dict([(t, cost) for cost, t in zip(c, trips)])
c = matrix(c)
nlog.debug("Constraints: %f\t%f" , maxdisk, maxoverhead)
for (r, op, s), pqcost in zip(trips, G1p):
# if 'CumO' not in str(op): continue
nlog.debug('%s\t%s\t%.15f\t%f\t%f\t%.15f', op, str(s).ljust(25),
pqcost,
mp.get_disk(op,s),
mp.get_provcost(op, s),
d[(r, op, s)])
strategies, torm = nlp_exec_cvx(c, ops, matstrats, trips, G, h, A, b)
for mapop in mapops:
strategies[mapop] = [Strat.full()]
return strategies, torm
def gen_nlp(self, maxdisk=500, maxoverhead=0.5):
strategies = run_nlp(Stats.instance(), self, maxdisk, maxoverhead)
for op, strats in strategies.items():
wlog.info("%s\t%s", op, ', '.join(map(str, strats)))
wlog.info("\n")
return strategies
