def classify_and_eval_h_rand(test_file):
def add_l2r_score_rand(infile, outfile):
from itertools import izip, dropwhile
from random import randint, seed
seed()
with open(infile) as fin, open(outfile, "w") as fout:
for job_str in dropwhile(swf_skip_hdr, fin):
new_score_str = str(randint(0, 1000000))
job = [u for u in job_str.strip().split()]
new_job_str = ' '.join(job[:-1] + [new_score_str])
fout.write(new_job_str + '\n')
print "[Simulating the test file using L2R..]"
test_l2r_fn = "%s_with_score.swf" % test_file.split('.')[0]
add_l2r_score_rand(test_file, test_l2r_fn)
# Simulate the file after l2r
config["scheduler"]["name"] = 'l2r_maui_scheduler'
config["weights"] = (0, 0, 0, 0, 0, 0, 1)
config["input_file"] = test_l2r_fn
config["output_swf"] = "%s_rnd_sim.swf" % test_l2r_fn.split('.')[0]
parse_and_run_simulator(config)
l2r_bsld = PerfMeasure(config["output_swf"])
l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max = l2r_bsld.all()
print "+ [RND] Average Bounded-Slowdown:", l2r_bsld_avg
print "+ [RND] Median Bounded-Slowdown:", l2r_bsld_med
print "+ [RND] Max Bounded-Slowdown:", l2r_bsld_max
return (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)
def simulate_scheduler(path, sch_name):
config["scheduler"]["name"] = sch_name # 'easy_sjbf_scheduler'
config["input_file"] = path
config["output_swf"] = rel("%s_%s.out" % (simple_name(path), sch_name))
parse_and_run_simulator(config)
return config["output_swf"]
def simulate_scheduler(path, sch_name):
config["scheduler"]["name"] = sch_name # 'easy_sjbf_scheduler'
config["input_file"] = path
config["output_swf"] = rel("%s_%s.out" % (simple_name(path), sch_name))
parse_and_run_simulator(config)
return config["output_swf"]
def classify_and_eval_h_rand(test_file):
def add_l2r_score_rand(infile, outfile):
from itertools import izip, dropwhile
from random import randint, seed
seed()
with open(infile) as fin, open(outfile, "w") as fout:
for job_str in dropwhile(swf_skip_hdr, fin):
new_score_str = str(randint(0, 1000000))
job = [u for u in job_str.strip().split()]
new_job_str = ' '.join(job[:-1] + [new_score_str])
fout.write(new_job_str + '\n')
print "[Simulating the test file using L2R..]"
test_l2r_fn = "%s_with_score.swf" % test_file.split('.')[0]
add_l2r_score_rand(test_file, test_l2r_fn)
# Simulate the file after l2r
config["scheduler"]["name"] = 'l2r_maui_scheduler'
config["weights"] = (0, 0, 0, 0, 0, 0, 1)
config["input_file"] = test_l2r_fn
config["output_swf"] = "%s_rnd_sim.swf" % test_l2r_fn.split('.')[0]
parse_and_run_simulator(config)
l2r_bsld = PerfMeasure(config["output_swf"])
l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max = l2r_bsld.all()
print "+ [RND] Average Bounded-Slowdown:", l2r_bsld_avg
print "+ [RND] Median Bounded-Slowdown:", l2r_bsld_med
print "+ [RND] Max Bounded-Slowdown:", l2r_bsld_max
return (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)
def simulate_scheduler_in_memory(path, sch_name):
config["scheduler"]["name"] = sch_name # 'easy_sjbf_scheduler'
config["input_file"] = path
parse_and_run_simulator(config)
jobs = config["terminated_jobs"]
del config["terminated_jobs"]
return jobs
def simulate_scheduler_in_memory(path, sch_name): config["scheduler"]["name"] = sch_name # 'easy_sjbf_scheduler' config["input_file"] = path parse_and_run_simulator(config) jobs = config["terminated_jobs"] del config["terminated_jobs"] return jobs
def simulate(path):
out_swf = []
for idx, w in enumerate(weights_options):
config["weights"] = w
config["input_file"] = path
config["output_swf"] = rel("%s_%d.swf" % (simple_name(path), idx))
parse_and_run_simulator(config)
out_swf.append(config["output_swf"])
return out_swf
def simulate(path):
out_swf = []
for idx, w in enumerate(weights_options):
config["weights"] = w
config["input_file"] = path
config["output_swf"] = rel("%s_%d.swf" % (simple_name(path), idx))
parse_and_run_simulator(config)
out_swf.append(config["output_swf"])
return out_swf
def simulate(path):
config = OnlineL2RMauiScheduler.config
out_swf = []
for idx, w in enumerate(OnlineL2RMauiScheduler.weights_options):
config["weights"] = w
config["input_file"] = path
config["output_swf"] = "%s_%d_%d.swf" % (path.split('.')[0], self.curr_part_num, idx)
parse_and_run_simulator(config)
out_swf.append(config["output_swf"])
# out_swf.append(config["terminated_jobs"])
# gc.collect() # to ensure that the output files are closed
return out_swf
def simulate(path):
config = AdaptiveMauiScheduler.config
out_swf = []
for idx, w in enumerate(AdaptiveMauiScheduler.weights_options):
config["weights"] = w
config["input_file"] = path
config["output_swf"] = "%s_%d_%d.swf" % (path.split('.')[0], self.curr_part_num, idx)
parse_and_run_simulator(config)
out_swf.append((w, config["output_swf"]))
# out_swf.append(config["terminated_jobs"])
# gc.collect() # to ensure that the output files are closed
return out_swf
def launchExpe(options, worker_id): with expe_counter.get_lock(): expe_counter.value += 1 myid = expe_counter.value #if not ( os.path.isfile(options["output_swf"]) ): print bcolors.WARNING+"Start expe "+str(myid)+" on w"+str(worker_id)+ bcolors.ENDC+" : "+str(options) error = False tempout = sys.stdout sys.stdout = open(options["output_swf"]+".out", 'w') sys.stderr = sys.stdout try: parse_and_run_simulator(options) except Exception,e: print "Exception: "+str(e) error = str(e)
def launchExpe(options, worker_id):
with expe_counter.get_lock():
expe_counter.value += 1
myid = expe_counter.value
#if not ( os.path.isfile(options["output_swf"]) ):
print bcolors.WARNING + "Start expe " + str(myid) + " on w" + str(
worker_id) + bcolors.ENDC + " : " + str(options)
error = False
tempout = sys.stdout
sys.stdout = open(options["output_swf"] + ".out", 'w')
sys.stderr = sys.stdout
try:
parse_and_run_simulator(options)
except Exception, e:
print "Exception: " + str(e)
error = str(e)
def launchExpe(options): myid = next_id() if not ( skip_always_done and os.path.isfile(options["output_swf"]) ): print bcolors.WARNING+"Start expe "+str(myid)+ bcolors.ENDC+" : "+str(options) error = False tempout = sys.stdout sys.stdout = open(options["output_swf"]+".out", 'w') sys.stderr = sys.stdout try: parse_and_run_simulator(options) except Exception,e: print "Exception: "+str(e) error = str(e) sys.stdout = tempout if not error: print bcolors.OKBLUE+"End epxe "+str(myid)+ bcolors.ENDC else: print bcolors.FAIL+"ERROR on "+str(myid)+": "+str(e)+ bcolors.ENDC
def classify_and_eval_h(test_file):
global min_max
print "[Creating the ML testing file..]"
mat = extract_columns(test_file, indices)
mat = normalize_mat(mat, min_max)
features = convert_to_ml_format(mat, 0)
write_lines_to_file(rel(test_fn), features)
print "[Classifying/Testing..]"
batchL2Rlib.classify(test_fn, model_fn, score_fn)
print "[Simulating the test file using L2R..]"
# add the l2r to the test log as the think time.
test_l2r_fn = "%s_with_score.swf" % test_file.split('.')[0]
add_l2r_score(test_file, rel(score_fn), test_l2r_fn)
# Simulate the file after l2r
config["scheduler"]["name"] = 'l2r_maui_scheduler'
config["weights"] = (0, 0, 0, 0, 0, 0, 1)
config["input_file"] = test_l2r_fn
config["output_swf"] = "%s_sim.swf" % test_l2r_fn.split('.')[0]
parse_and_run_simulator(config)
l2r_bsld = PerfMeasure(config["output_swf"])
l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max = l2r_bsld.all()
print "+ [L2R] Average Bounded-Slowdown:", l2r_bsld_avg
print "+ [L2R] Median Bounded-Slowdown:", l2r_bsld_med
print "+ [L2R] Max Bounded-Slowdown:", l2r_bsld_max
#print "+ [L2R] System Utilization:", compute_utilisation(config["output_swf"])
res_data.extend(
[str(u) for u in (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)])
res_data.append(' ')
return (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)
def classify_and_eval_h(test_file):
global min_max
print "[Creating the ML testing file..]"
mat = extract_columns(test_file, indices)
mat = normalize_mat(mat, min_max)
features = convert_to_ml_format(mat, 0)
write_lines_to_file(rel(test_fn), features)
print "[Classifying/Testing..]"
batchL2Rlib.classify(test_fn, model_fn, score_fn)
print "[Simulating the test file using L2R..]"
# add the l2r to the test log as the think time.
test_l2r_fn = "%s_with_score.swf" % test_file.split('.')[0]
add_l2r_score(test_file, rel(score_fn), test_l2r_fn)
# Simulate the file after l2r
config["scheduler"]["name"] = 'l2r_maui_scheduler'
config["weights"] = (0, 0, 0, 0, 0, 0, 1)
config["input_file"] = test_l2r_fn
config["output_swf"] = "%s_sim.swf" % test_l2r_fn.split('.')[0]
parse_and_run_simulator(config)
l2r_bsld = PerfMeasure(config["output_swf"])
l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max = l2r_bsld.all()
print "+ [L2R] Average Bounded-Slowdown:", l2r_bsld_avg
print "+ [L2R] Median Bounded-Slowdown:", l2r_bsld_med
print "+ [L2R] Max Bounded-Slowdown:", l2r_bsld_max
#print "+ [L2R] System Utilization:", compute_utilisation(config["output_swf"])
res_data.extend([str(u) for u in (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)])
res_data.append(' ')
return (l2r_bsld_avg, l2r_bsld_med, l2r_bsld_max)
#! /usr/bin/env python2
from run_simulator import parse_and_run_simulator
options = {
"scheduler" : {
"name":'TBD'
},
"input_file": '../../../data/CEA-curie_sample/original_swf/log.swf',
"num_processors" : 80640,
"stats" : False,
"output_swf" : 'TBD'
}
for sched in ('easy_backfill_scheduler', "easy_sjbf_scheduler"):
options["scheduler"]["name"] = sched
options["output_swf"] = "res_"+sched+".swf"
parse_and_run_simulator(options)
sim_statistics(log_path, 'online_l2r_maui_scheduler')
# simulate with all the schedulers
if arguments["-a"]:
sim_statistics(log_path, 'adaptive_maui_scheduler')
# simulate with diff weights
if arguments["-w"]:
if redirect_sim_output: sys.stdout = open('/dev/null', 'w')
vals = []
try:
for w in weights_options:
config["weights"] = w
config["input_file"] = log_path
parse_and_run_simulator(config)
sim_jobs = config["terminated_jobs"]
del config["terminated_jobs"]
obj_fun_val = PerfMeasure(jobs=sim_jobs).average()
vals.append((w, obj_fun_val))
print >> sys.__stdout__, vals[-1]
except:
pass
vals.sort(key=lambda u: u[1])
print >> sys.__stdout__
for i in range(min(10, len(vals))):
print >> sys.__stdout__, vals[i]
if redirect_sim_output: sys.stdout = sys.__stdout__
# train offline
#! /usr/bin/env python2
from run_simulator import parse_and_run_simulator
options = {
"scheduler": {
"name": 'TBD'
},
"input_file": '../../../data/CEA-curie_sample/original_swf/log.swf',
"num_processors": 80640,
"stats": False,
"output_swf": 'TBD'
}
for sched in ('easy_backfill_scheduler', "easy_sjbf_scheduler"):
options["scheduler"]["name"] = sched
options["output_swf"] = "res_" + sched + ".swf"
parse_and_run_simulator(options)
sim_statistics(log_path, 'online_l2r_maui_scheduler')
# simulate with all the schedulers
if arguments["-a"]:
sim_statistics(log_path, 'adaptive_maui_scheduler')
# simulate with diff weights
if arguments["-w"]:
if redirect_sim_output: sys.stdout = open('/dev/null', 'w')
vals = []
try:
for w in weights_options:
config["weights"] = w
config["input_file"] = log_path
parse_and_run_simulator(config)
sim_jobs = config["terminated_jobs"]
del config["terminated_jobs"]
obj_fun_val = PerfMeasure(jobs=sim_jobs).average()
vals.append((w, obj_fun_val))
print >> sys.__stdout__, vals[-1]
except:
pass
vals.sort(key= lambda u: u[1])
print >> sys.__stdout__
for i in range(min(10, len(vals))):
print >> sys.__stdout__, vals[i]
if redirect_sim_output: sys.stdout = sys.__stdout__
# train offline
