def pre_submit_plan(host, deploy):
for vm in deploy['workers']:
requests.get(url(host, vm[3]) + "sysgc")
time.sleep(1)
if 'myjava' in deploy['name']:
eused, oused = jvm.get(host, vm[4], ['eused', 'oused']).strip().split('|')[:2]
jvm.send('localhost', vm[4], ['smax=0', 'doneinit'])
def get_tpch_plan(self, qid):
myria_utils.shutdown(self.ins)
myria_utils.launch(self.ins, deploy)
time.sleep(1)
jvm.send(self.ins, deploy['workers'][0][4],
['emax=max', 'smax=0', 'omax=max', 'doneinit'])
time.sleep(1)
ingest_tpch.ingesttpchdata(self.ins)
with open('/home/jwang/project/tpch-dbgen/%s.json' % qid) as data_file:
return json.load(data_file)
def get_rss_gen_max(host, deploy):
launch(host, deploy)
time.sleep(1)
requests.get(url(host, deploy['workers'][0][3]) + "sysgc")
time.sleep(1)
ret = jvm.get(host, deploy['workers'][0][4], ['eused', 'oused']).strip().split('|')
jvm.send(host, deploy['workers'][0][4], ['emax=%s' % ret[0], 'smax=0', 'omax=%s' % ret[1], 'doneinit'])
time.sleep(1)
initrss = int(jvm.get(host, deploy['workers'][0][4], ['rss'])[:-1])
emax, omax = jvm.get(host, deploy['workers'][0][4], ['emax', 'omax']).strip().split('|')[:2]
return (initrss, emax, omax)
def submit_plan(plan, hostname):
myria_utils.drop_cache(hostname)
myria_utils.shutdown(hostname)
time.sleep(1)
myria_utils.launch(hostname, deploy)
time.sleep(1)
jvm.send(hostname, WORKER_JVM,
['emax=max', 'smax=0', 'omax=max', 'doneinit'])
time.sleep(1)
requests.get(myria_utils.url(hostname, WORKER_REST) + "sysgc")
time.sleep(2)
r = requests.post(myria_utils.url(hostname, MASTER_REST) + "query",
data=json.dumps(plan),
headers={'Content-Type': 'application/json'})
print r.text
qid = str(r.json()["queryId"])
print "running query %s plan %s on %s" % (qid, json.dumps(plan), hostname)
return qid
def adjust(scheme, conf, elapsed, memincre):
running_before = len(running)
global gchistory
global killtime
for i in scheme:
print i, scheme[i]
decision = scheme[i][0]
if conf['debug']: # and decision in ['FGC_NOPM','FGC_PM','YGC']:
s = history[i][-1]
print '* state transfering for ', i, s['qid'], s['timestamp']
print ' ', 'opstats', s['opstats']
print ' ', 'eden used cap', s['eused'], s[
'ecap'], 'old used cap', s['oused'], s['ocap'], 'blocked', s[
'blocked_count'], s['blocked_sleep'], 'gcactive', s[
'gcactive'], 'cpu', s['cpu'], 'lastgcsaved', s[
'lastgcsaved']
print ' ', 'predict', 'yreal', s['yreal'], 'ylsize', s[
'ylsize'], 'ydsize', s['ydsize'], 'oreal', s[
'oreal'], 'olsize', s['olsize'], 'odsize', s[
'odsize'], 'next_yused', s['next_yused']
print ' ', i, scheme[i]
if len(history[i]) >= 2 and 'next_yused' in history[i][-2]:
print 'grow', i, s['yused'], history[i][-2]['next_yused']
if decision == 'OOM':
for vm in deploys[i[1]]['workers']:
if vm in running:
collect_log_thread(i[1], vm, running[vm], 'kill').start()
print 'kill', vm
del running[vm]
append_resub(conf, i[1], running_before)
check_resubmit(conf)
killtime.append(str(elapsed))
continue
if decision in ['NOOP', 'INGC']: continue
args = ['emax=%d' % scheme[i][1], 'omax=%d' % scheme[i][2]]
if decision in ['FGC_NOPM', 'FGC_PM', 'YGC']:
if i not in gchistory: gchistory[i] = []
gchistory[i].append(history[i][-1])
if decision.startswith('FGC'): decision += '_' + str(memincre)
args.append(decision.lower())
running[i]['opstats_beforegc'] = parse_opstats(
running[i]['opstats_latest'] if 'opstats_latest' in
running[i] else '')
jvm.send('localhost', i[4], args)