def stats(file):
with open("../dump/" + str(file) + ".pickle", "r") as f:
results = pickle.load(f)
name = file.split('_')[0]
test = []
for key in results:
a = key.split('_')[0]
try:
b = key.split('_')[1]
except:
b = '0'
if b == "2" and name == "UPDATE":
if a == "POS":
a = "UPDATE_POS"
elif a == "UPDATE":
a = "UPDATE_ALL"
tmp = []
for r in results[key]:
tmp.append(r['x'][-1])
print(a + ": max %d" % max(tmp))
test.append([a] + tmp)
elif name != "UPDATE":
tmp = []
for r in results[key]:
tmp.append(r['x'][-1])
test.append([a] + tmp)
print(a + ": max %d" % max(tmp))
rdivDemo(test, isLatex=True)
set_trace()
def sum_relative():
with open("./dump/result.pickle", "r") as handle:
results = pickle.load(handle)
result = {}
for metrics in results:
result[metrics] = {}
for t in results[metrics]:
if t == 'A2':
continue
result[metrics][t] = np.array(results[metrics][t]) / np.array(
results[metrics]['A2'])
results = result
summary = {}
for metrics in results:
summary[metrics] = {}
for t in results[metrics]:
summary[metrics][t] = {
'median':
np.median(results[metrics][t]),
'iqr':
np.percentile(results[metrics][t], 75) -
np.percentile(results[metrics][t], 25)
}
print("summary:")
print(summary)
for metrics in results:
print(metrics)
if metrics == "X50":
rdivDemo(results[metrics], bigger_better=False)
else:
rdivDemo(results[metrics], bigger_better=False)
def process():
rc('text', usetex=True)
folder = "./PickleLocker/"
mmre = pickle.load(open(folder + "mmre-based.p"))
rank = pickle.load(open(folder + "rank-based.p"))
epal = pickle.load(open(folder + "epal.p"))
al = pickle.load(open(folder + "al-based.p"))
al_2 = pickle.load(open(folder + "al-based-2.p"))
al_3 = pickle.load(open(folder + "al-based-3.p"))
pop0 = pickle.load(open(folder + "pop0-based.p"))
files = mmre.keys()
for i, file in enumerate(sorted(files)):
lists = list()
# lists.append(["MMRE-Prog"] + mmre[file]['gen_dist'])
# lists.append(["Rank-Prog"] + rank[file]['gen_dist'])
lists.append(["ePAL-0.01"] + epal[file][0.01]['gen_dist'])
lists.append(["ePAL-0.02"] + epal[file][0.02]['gen_dist'])
lists.append(["ePAL-0.04"] + epal[file][0.04]['gen_dist'])
lists.append(["ePAL-0.08"] + epal[file][0.08]['gen_dist'])
lists.append(["ePAL-0.12"] + epal[file][0.12]['gen_dist'])
lists.append(["ePAL-0.16"] + epal[file][0.16]['gen_dist'])
lists.append(["ePAL-0.2"] + epal[file][0.2]['gen_dist'])
lists.append(["ePAL-0.3"] + epal[file][0.3]['gen_dist'])
lists.append(["AL1"] + al[file]['gen_dist'])
lists.append(["\hl{AL2}"] + al_2[file]['gen_dist'])
lists.append(["AL3"] + al_3[file]['gen_dist'])
lists.append(["\colorbox{blue}{pop0}"] + pop0[file]['gen_dist'])
rdivDemo("SS" + str(i + 1),file.split('/')[-1].split('.')[0].replace('_', '\_'), lists, globalMinMax=False, isLatex=True)
def get_reults(self, bellwethers=None, n_reps=12):
data_path = os.path.realpath("./data")
projects = get_all_projects(data_path)
for project in projects:
print(project.name.upper())
files = project.files()
results_0 = []
for transfer in [Pooyan, Baseline, Waterloo]:
results_1 = [transfer.__doc__.upper()]
for source_name, source_conf in files.iteritems():
for target_name, target_conf in files.iteritems():
if not source_name == target_name:
if transfer.__doc__ == "baseline":
if source_name in bellwethers[project.name]:
results_1.extend([
transfer.learner(
source_conf, target_conf)
for _ in xrange(n_reps)
])
else:
results_1.extend([
transfer.learner(source_conf, target_conf)
for _ in xrange(n_reps)
])
results_0.append(results_1)
rdivDemo(results_0)
print("")
def bellwether(self):
everything = []
src = data(dataName=self.file, type=self.type)
self.test = createTbl(src.test, isBin=True)
table_rows = [["Dataset", "ED", "G2", "Pd", "Pf"]]
if len(src.train) == 1:
train = src.train[0]
else:
train = src.train
header = [" "]
# onlyMe = [self.file]
val = []
for file in train:
fname = file[0].split('/')[-2]
e = [fname]
header.append(fname)
self.train = createTbl(file, isBin=True)
actual = Bugs(self.test)
for _ in xrange(10):
predicted = rforest(
self.train,
self.test,
tunings=self.param,
smoteit=True)
p_buggy = [a for a in ABCD(before=actual, after=predicted).all()]
e.append(p_buggy[1].stats()[-2])
# val.append([fname, "%0.2f" % p_buggy[1].stats()[-2], "%0.2f" % p_buggy[1].stats()[-3]
# , "%0.2f" % p_buggy[1].stats()[0], "%0.2f" % p_buggy[1].stats()[1]])
everything.append(e)
rdivDemo(everything, isLatex=True)
def run(): G = [] for _ in xrange(1): G.append(haupt()) G.insert(0, 'RF ') # print base.main()+[G] sk.rdivDemo(base.main() + [G]) set_trace()
def run():
G = []
for _ in xrange(1):
G.append(haupt())
G.insert(0, 'RF ')
# print base.main()+[G]
sk.rdivDemo(base.main() + [G])
set_trace()
def _test():
for mdl in [POM3, XOMO]:
print('## %s \n\n' % (mdl.__doc__))
ndep = 4 if mdl == XOMO else 3
random.seed(0)
for n in xrange(ndep):
print('#### %s \n```' % (mdl().depen[n][2:]))
R = [r for r in learner(mdl, n, reps=28)]
rdivDemo(R, isLatex=True)
print('```')
def printsk(res):
"Now printing only g"
tosh = []
for p in res:
dat1, dat2, dat3, dat4 = res[p]
tosh.append([dat1[0][0]] + [k[-1] for k in dat1])
tosh.append([dat2[0][0]] + [k[-1] for k in dat2])
tosh.append([dat3[0][0]] + [k[-1] for k in dat3])
tosh.append([dat4[0][0]] + [k[-1] for k in dat4])
rdivDemo(tosh, isLatex = False)
def printsk(res):
"Now printing only g"
tosh = []
for p in res:
dat1, dat2, dat3, dat4 = res[p]
tosh.append([dat1[0][0]] + [k[-1] for k in dat1])
tosh.append([dat2[0][0]] + [k[-1] for k in dat2])
tosh.append([dat3[0][0]] + [k[-1] for k in dat3])
tosh.append([dat4[0][0]] + [k[-1] for k in dat4])
rdivDemo(tosh, isLatex=False)
def aeeem():
print("AEEEM\n------\n```")
for file in ["EQ", "JDT", "ML", "PDE"]:
print('### ' + file)
# makeDataSets(file, type='aeeem', tune=False).barak09()
E = []
E.append(simulate(file, type='aeeem').turhan09())
E.append(simulate(file, type='aeeem').turhan11())
E.append(simulate(file, type='aeeem').bellwether())
rdivDemo(E, isLatex=True)
print('```')
def relink():
print("Relink\n------\n```")
for file in ["Apache", "Zxing"]:
print('### ' + file)
# makeDataSets(file, type='relink', tune=False).barak09()
E = []
E.append(simulate(file, type='relink').turhan09())
E.append(simulate(file, type='relink').turhan11())
E.append(simulate(file, type='relink').bellwether())
rdivDemo(E, isLatex=True)
print('```')
def jur():
print("Jureczko\n------\n```")
for file in [ 'log4j', 'ant', 'camel', 'ivy', 'jedit'
, 'poi', 'velocity', 'xalan', 'xerces']:
print('### ' + file)
E = []
E.append(simulate(file, type='jur').turhan09())
E.append(simulate(file, type='jur').turhan11())
E.append(simulate(file, type='jur').bellwether())
rdivDemo(E, isLatex=True)
print('```')
def tiles(self):
num_obs = len(self.generations[0][0].objectives)
for i in range(num_obs):
obj_gens=[]
for gen, pop in enumerate(self.generations):
if gen % self.settings.gen_step != 0:
continue
objs = ["gen"+str(gen)+"_f"+str(i+1)]
for point in pop:
objs.append(point.objectives[i])
obj_gens.append(objs)
sk.rdivDemo(obj_gens)
def main():
files = glob("./Processed/*.p")
for file in files:
system_name = file.split('/')[-1].split('.p')[0]
d = pickle.load(open(file))
outer = []
for src, val in d.iteritems():
inner = [src]
for _, v in val.iteritems():
inner.extend(v['rank'])
outer.append(inner)
rdivDemo(outer)
def tiles(self):
num_obs = len(self.generations[0][0].objectives)
for i in range(num_obs):
obj_gens = []
for gen, pop in enumerate(self.generations):
if gen % self.settings.gen_step != 0:
continue
objs = ["gen" + str(gen) + "_f" + str(i + 1)]
for point in pop:
objs.append(point.objectives[i])
obj_gens.append(objs)
sk.rdivDemo(obj_gens)
def _test(file='cs'):
filepath = os.path.dirname(os.path.abspath(__file__))+"/dataset/"
thres = [0.02, 0.05]
#sys.stdout = Logger("yourlogfilename.txt")
issel = ["tfidf", "tf"]
isshingle=["no_shingle","bigram","trigram"]
filenamelist = ['cs', 'diy', 'academia', 'judaism', 'photo', 'rpg', 'scifi', 'ux']
for i in range(10):
filenamelist.append('SE'+str(i))
print("started")
F_final = {}
for filename in filenamelist:
print(filename)
temp_file = {}
for feature in issel:
temp_feature={}
for is_shingle in isshingle:
temp_feature[is_shingle] = cross_val(filename=filename, filepath=filepath, filetype='.txt', thres=thres,
folds=5, feature=feature, is_shingle=is_shingle,
n_feature=1000)
#temp_file=temp_feature
temp_file[feature]=temp_feature
F_final[filename] = temp_file
#"Scott-knot"
#print(filename + ":")
print ("\n")
tmp = []
for feature in issel:
for is_shingle in isshingle:
tmp.append([feature+"_"+is_shingle]+temp_file[feature][is_shingle])
print (rdivDemo(tmp))
print ("\n")
'''with open('result.pickle', 'rb') as handle:
def rdiv():
lst = []
def striplines(line):
listedline = line.strip().split(',') # split around the = sign
listedline[0] = listedline[0][2:-1]
lists = [listedline[0]]
for ll in listedline[1:-1]:
lists.append(float(ll))
return lists
f = open('./dataCPM.txt')
for line in f:
lst.append(striplines(line[:-1]))
rdivDemo(lst, isLatex=False)
def rdiv():
lst = []
def striplines(line):
listedline = line.strip().split(',') # split around the = sign
listedline[0] = listedline[0][2:-1]
lists = [listedline[0]]
for ll in listedline[1:-1]:
lists.append(float(ll))
return lists
f = open('./dataCPM.txt')
for line in f:
lst.append(striplines(line[:-1]))
rdivDemo(lst, isLatex=False)
def sum_HPC():
with open("./result/result.pickle","r") as handle:
results = pickle.load(handle)
summary = {}
for metrics in results:
summary[metrics] = {}
for t in results[metrics]:
summary[metrics][t]={'median': np.median(results[metrics][t]), 'iqr': np.percentile(results[metrics][t],75)-np.percentile(results[metrics][t],25)}
print("summary:")
print(summary)
for metrics in results:
print(metrics)
if metrics == "X50":
rdivDemo(results[metrics],bigger_better=False)
else:
rdivDemo(results[metrics],bigger_better=False)
def run(decs=10,objs=2,iters=20):
era_data = []
model = DTLZ7(decs, objs)
for i in xrange(1,iters+1):
for optimizer in [MWS, SA, DE]:
era_val = [model.normalize_val(model.eval(val)) for val in optimizer(model)]
era_val.insert(0, str(optimizer.__name__) + str(i))
if((i==1)or(i==iters)):
era_data.append(era_val)
print(rdivDemo(era_data))
def type3(final_era, zero_era):
i=0
for final_opt, zero_opt in zip(final_era,zero_era):
losses = [ ]
length = min(len(final_opt), len(zero_opt))
for each in range(length):
final_cand = final_opt[each]
zero_cand = zero_opt[each]
num = (final_cand[1]-zero_cand[1])**2 + (final_cand[0]-zero_cand[0])**2
den = 2
rms_dist = math.sqrt(num/den)
#print rms_dist
losses.append(rms_dist)
for each in losses:
data[i].append(each)
i=i+1
#print data
rdivDemo(data)
def printResult(dataname, which_is_better, lst, file_name, goal_index):
def count_better(dicts):
temp = {}
learner_name = set(
[i[i.index("_") + 1:i.index(":")] for i in dicts.keys()])
for key, val in dicts.iteritems():
if "Y-Def" in key and ("Tuned_" in key or "Grid_" in key):
temp[key] = np.median(val)
for each in learner_name:
tune = "Tuned_" + each + ": Y-Def"
grid = "Grid_" + each + ": Y-Def"
if temp[tune] >= temp[grid]:
which_is_better[tune] = which_is_better.get(tune, 0) + 1
else:
which_is_better[grid] = which_is_better.get(grid, 0) + 1
def myrdiv(d):
stat = []
for key, val in d.iteritems():
val.insert(0, key)
stat.append(val)
return stat
print("\n" + "+" * 20 + "\n DataSet: " + dataname + "\n" + "+" * 20)
obj = ["pd", "pf", "prec", "f", "g", "auc"]
for j, k in enumerate(obj):
express = "\n" + "*" * 10 + " " + k + " " + "*" * 10
print(express)
writefile(file_name, express)
# pdb.set_trace()
if j == goal_index:
count_better(lst[j])
rdivDemo(file_name, myrdiv(lst[j]))
out_better = (
"\n In terms of " + str(goal_index) + " : the times of better "
"tuners are" + str(
which_is_better))
print(out_better)
writefile(file_name, out_better)
writefile(file_name,
"End time :" + strftime("%Y-%m-%d %H:%M:%S") + "\n" * 2)
print("\n")
def printResult(dataname, which_is_better, lst, file_name, goal_index):
def count_better(dicts):
temp = {}
learner_name = set(
[i[i.index("_") + 1:i.index(":")] for i in dicts.keys()])
for key, val in dicts.iteritems():
if "Y-Def" in key and ("Tuned_" in key or "Grid_" in key):
temp[key] = np.median(val)
for each in learner_name:
tune = "Tuned_" + each + ": Y-Def"
grid = "Grid_" + each + ": Y-Def"
if temp[tune] >= temp[grid]:
which_is_better[tune] = which_is_better.get(tune, 0) + 1
else:
which_is_better[grid] = which_is_better.get(grid, 0) + 1
def myrdiv(d):
stat = []
for key, val in d.iteritems():
val.insert(0, key)
stat.append(val)
return stat
print("\n" + "+" * 20 + "\n DataSet: " + dataname + "\n" + "+" * 20)
obj = ["pd", "pf", "prec", "f", "g"]
for j, k in enumerate(obj):
express = "\n" + "*" * 10 + " " + k + " " + "*" * 10
print(express)
writefile(file_name, express)
# pdb.set_trace()
# if j == goal_index:
# count_better(lst[j])
rdivDemo(file_name, myrdiv(lst[j]))
out_better = (
"\n In terms of " + str(goal_index) + " : the times of better "
"tuners are" + str(
which_is_better))
print(out_better)
writefile(file_name, out_better)
writefile(file_name,
"End time :" + strftime("%Y-%m-%d %H:%M:%S") + "\n" * 2)
print("\n")
def _doCrossVal():
cv_acc = []
cv_auc = []
cv_md = []
for name in ['Apache', 'SQL', 'BDBC', 'BDBJ', 'X264', 'LLVM']:
a, _, __ = fileHandler().crossval(name, k=5)
cv_acc.append(a)
# cv_auc.append(b)
# cv_md.append(c)
# print(r"""\documentclass{article}
# \usepackage{colortbl}
# \usepackage{fullpage}
# \usepackage[table]{xcolor}
# \usepackage{picture}
# \newcommand{\quart}[4]{\begin{picture}(100,6)
# {\color{black}\put(#3,3){\circle*{4}}\put(#1,3){\line(1,0){#2}}}\end{picture}}
# \begin{document}
# """)
# print(r"\subsubsection*{Accuracy}")
rdivDemo(cv_acc, isLatex=False)
def _doCrossVal():
cv_acc = []
cv_auc = []
cv_md = []
for name in ['Apache', 'SQL', 'BDBC', 'BDBJ', 'X264', 'LLVM']:
a, _, __ = fileHandler().crossval(name, k=5)
cv_acc.append(a)
# cv_auc.append(b)
# cv_md.append(c)
# print(r"""\documentclass{article}
# \usepackage{colortbl}
# \usepackage{fullpage}
# \usepackage[table]{xcolor}
# \usepackage{picture}
# \newcommand{\quart}[4]{\begin{picture}(100,6)
# {\color{black}\put(#3,3){\circle*{4}}\put(#1,3){\line(1,0){#2}}}\end{picture}}
# \begin{document}
# """)
# print(r"\subsubsection*{Accuracy}")
rdivDemo(cv_acc, isLatex=False)
def compare(self, data_pairs):
data_path = os.path.realpath("./data")
projects = get_all_projects(data_path)
for project in projects:
print(project.name.upper())
files = project.files()
best, worst = data_pairs[project.name]
rest = [
dframe for fname, dframe in files.iteritems()
if fname not in data_pairs[project.name]
]
best_results = ["best"]
worst_results = ["worst"]
for tgt in rest:
best_results.extend(
[Pooyan.learner(files[best], tgt) for _ in xrange(1)])
worst_results.extend(
[Pooyan.learner(files[worst], tgt) for _ in xrange(1)])
rdivDemo([best_results, worst_results])
set_trace()
def summary(filename):
with open("../dump/" + str(filename) + ".pickle", "r") as f:
results = pickle.load(f)
test = []
total = results['linear'][0]['x'][-1]
wss95 = []
for key in results:
# if 'M' in key:
# continue
tmp = []
tmp_wss = []
for r in results[key]:
if key == 'linear':
tmp.append(int(r['x'][-1] * 0.95))
tmp_wss.append(0.05 - (total - int(r['x'][-1] * 0.95)) / total)
else:
tmp.append(r['x'][-1])
tmp_wss.append(0.05 - (total - r['x'][-1]) / total)
test.append([key] + tmp)
wss95.append([key] + tmp_wss)
rdivDemo(test, isLatex=True)
set_trace()
rdivDemo(wss95, isLatex=False)
def _test(file='cs'):
filepath = os.path.dirname(os.path.abspath(__file__)) + "/dataset/"
thres = [0.02, 0.05]
#sys.stdout = Logger("yourlogfilename.txt")
issel = ["tfidf", "tf"]
isshingle = ["no_shingle", "bigram", "trigram"]
filenamelist = [
'cs', 'diy', 'academia', 'judaism', 'photo', 'rpg', 'scifi', 'ux'
]
for i in range(10):
filenamelist.append('SE' + str(i))
print("started")
F_final = {}
for filename in filenamelist:
print(filename)
temp_file = {}
for feature in issel:
temp_feature = {}
for is_shingle in isshingle:
temp_feature[is_shingle] = cross_val(filename=filename,
filepath=filepath,
filetype='.txt',
thres=thres,
folds=5,
feature=feature,
is_shingle=is_shingle,
n_feature=1000)
#temp_file=temp_feature
temp_file[feature] = temp_feature
F_final[filename] = temp_file
#"Scott-knot"
#print(filename + ":")
print("\n")
tmp = []
for feature in issel:
for is_shingle in isshingle:
tmp.append([feature + "_" + is_shingle] +
temp_file[feature][is_shingle])
print(rdivDemo(tmp))
print("\n")
'''with open('result.pickle', 'rb') as handle:
def _test(file='ant'):
for file in ['ivy', 'jedit', 'lucene', 'poi', 'ant']:
print('## %s\n```' % (file))
R = [r for r in run(dataName=file, reps=1).go()]
rdivDemo(R, isLatex=False)
print('```')
__author__ = 'Sushil'
import glob
import pickle
from sk import rdivDemo
if __name__ == '__main__':
rDiv_ip=[]
for file in glob.glob('outputs/*lst'):
with open(file, 'rb') as f:
my_list = pickle.load(f)
rDiv_ip.append(my_list)
rdivDemo(rDiv_ip)
from sk import rdivDemo
results = {'treatment1': [1, 1, 1], 'treatment2': [1, 2, 1]}
rdivDemo(results, isLatex=False)
print "Early Termination " + str(k) + " : " + str(eras)
return previous_era
# print("\nBest Solution : " + str(best_sol))
# print("Best Energy : " + str(model.normalize_val(model.eval(best_sol))))
return previous_era
if __name__ == '__main__':
era_collection = []
text = ["MWS", "SA", "DE"]
ct = 0
model = DTLZ7(10, 2)
i = 0
for _ in xrange(0, 20):
i += 1
for optimizer in [
max_walk_sat, simulated_annealing, differential_evolution
]:
era_val = [
model.normalize_val(model.eval(val))
for val in optimizer(model)
]
era_val.insert(0, text[ct % 3] + str(i))
era_collection.append(era_val)
ct += 1
# print era_collection
print rdivDemo(era_collection)
changed = True
new = x + f*(y - z)
out.have[d] = max(0.0, min(new, 1.0)) # keep in range
if not changed:
d = self.a(self.model.decisions())
out.have[d] = two.have[d]
out.score = self.model.energy(out.have) # remember to score it
out.energy = self.model.energy(out.have)
return out
model = DTLZ7(10, 2)
eras=[]
for i in xrange(20):
for algorithm in ["MaxWalkSat", "SA", "DE"]:
if algorithm == "MaxWalkSat":
algo = MaxWalkSat("DTLZ7")
elif algorithm == "SA":
algo = SA("DTLZ7")
elif algorithm == "DE":
algo = DE("DTLZ7")
era = [model.normalize_val(model.eneergy(val)) for val in algo.run()]
era.insert(0, algorithm + str(i))
eras.append(era)
print era
print rdivDemo(eras)
from sk import rdivDemo
pickle_files = [f for f in os.listdir(".") if ".py" not in f]
content = pickle.load(open(pickle_files[0]))
problems = content.keys()
prob = {}
for problem in problems:
al2 = pickle.load(open('al2_XOMO.p'))
al = pickle.load(open('al_XOMO.p'))
mmre = pickle.load(open('mmre_XOMO.p'))
nsgaii = pickle.load(open('nsgaii_XOMO.p'))
rank = pickle.load(open('rank_XOMO.p'))
spea2 = pickle.load(open('spea2_XOMO.p'))
sway5 = pickle.load(open('SWAY5_XOMO.p'))
lists = list()
lists.append(['AL2'] + al2[problem]['gen_dist'])
lists.append(['AL'] + al[problem]['gen_dist'])
lists.append(['MMRE'] + mmre[problem]['gen_dist'])
lists.append(['NSGAII'] + nsgaii[problem]['gen_dist'])
lists.append(['Rank'] + rank[problem]['gen_dist'])
lists.append(['SPEA2'] + spea2[problem]['gen_dist'])
lists.append(['SWAY5'] + sway5[problem]['gen_dist'])
rdivDemo(problem.replace('_', '\_'),
"",
lists,
globalMinMax=False,
isLatex=True)
hv_st[model][objnum][decnum]=[]
hv_st[model][objnum][decnum].append(np.median(hv[model][objnum][decnum][tune]))
hv_st[model][objnum][decnum].append(iqr(hv[model][objnum][decnum][tune]))
tmp2.append(str(np.median(hv[model][objnum][decnum][tune]))+u"\u00b1"+str(iqr(hv[model][objnum][decnum][tune])))
tmp.append(tmp2)
col.append(str(objnum)+"_"+tune)
print("model: "+str(model))
print(pd.DataFrame(data = tmp, columns=decs ,index=col))
#print(pd.DataFrame.from_dict(hv_st[model]))
Better={}
for model in models:
Better[model]=[]
for objnum in objs:
tmp=[]
for decnum in decs:
isbetter=[["untuned"]+hv[model][objnum][decnum]['untuned']]
isbetter.append(["tuned"]+hv[model][objnum][decnum]['tuned'])
ranks=rdivDemo(isbetter)
tmp.append(ranks[0][0]!=ranks[1][0])
Better[model].append(tmp)
print("model: "+str(model))
print(pd.DataFrame(data = Better[model], columns=decs ,index=objs))
from DE import DE as de
from NSGA2 import NSGA2 as nsga2
from SPEA2 import SPEA2 as spea2
from IntegratedProjectModel_QA import Integrated_project_model_qa as model
from sk import rdivDemo
if __name__ == '__main__':
optimize = []
for optimizer in [de, nsga2, spea2]:
print "Optimizer: %s, " % optimizer.__name__, "Model: %s\n" % model.__name__
optimize.append([optimizer.__name__] + optimizer(model))
print ""
print optimize
print ""
print(rdivDemo(optimize))
import pandas as pd
import pickle
import numpy as np
from sk import rdivDemo
file = "./Processed/processed_100.p"
content = pickle.load(open(file))
temp = content['x264'][100]
files = temp.keys()
ll = []
for source in files:
targets = temp[source].keys()
l = [source]
for target in targets:
l.append(np.median(temp[source][target]))
ll.append(l)
rdivDemo(
'x264',
ll,
isLatex=False,
globalMinMax=True,
)
f_10_50 = pickle.load(open(locker + 'FlashB_50.p'))
f_15_50 = pickle.load(open(locker + 'FlashB_15_50.p'))
f_20_50 = pickle.load(open(locker + 'FlashB_20_50.p'))
f_25_50 = pickle.load(open(locker + 'FlashB_25_50.p'))
f_30_50 = pickle.load(open(locker + 'FlashB_30_50.p'))
names = [
'SS-A', 'SS-B', 'SS-C', 'SS-D', 'SS-E', 'SS-F', 'SS-G', 'SS-H', 'SS-I',
'SS-J', 'SS-K', 'SS-L'
] #, 'SS-M', 'SS-N', 'SS-O']
quality = 'evals'
for name in names:
e1 = ['epal-0.01'] + epal[name][0.01][quality]
e2 = ['epal-0.3'] + epal[name][0.3][quality]
# f1 = ['Flash30'] + [f[0] for f in f30[name][quality]]
f20 = ['Flash50'] + [f[0] for f in f_10_50[name][quality]]
# f3 = ['Flash75'] + [f[0] for f in f75[name][quality]]
# f4 = ['Flash100'] + [f[0] for f in f100[name][quality]]
f21 = ['Flash50'] + [f[0] for f in f_15_50[name][quality]]
f22 = ['Flash50'] + [f[0] for f in f_20_50[name][quality]]
f23 = ['Flash50'] + [f[0] for f in f_25_50[name][quality]]
f24 = ['Flash50'] + [f[0] for f in f_30_50[name][quality]]
print name
# data.append([name, mean(res['rank_diff']), mean(rnk['rank_diff']), mean(flsh['rank_diff'])])
# data.append([name, mean(res['evals']), mean(rnk['evals']), mean(flsh['evals']), sizes[name]*0.2])
rdivDemo(name, [e1, e2, f24], globalMinMax=False, isLatex=False)
print file, len(targets)
myFile = open(result_folder + system_name + '.csv', 'w')
writer = csv.writer(myFile)
heading = ['Target Files', '5', '10', '15', '20', '25', '30', '35', '40', 's5', 's10', 's15', 's20', 's25', 's30', 's35', 's40']
writer.writerow(heading)
for target in targets:
percs = sorted(dd[source][target].keys(), key=lambda x: int(x))
t = [target]
aux = []
for perc in percs:
print perc,
bw = ['bw'] + dd[source][target][perc]['bellwether']
tar = ['target'] + dd[source][target][perc]['target']
# print perc, bw, tar
ret = rdivDemo('a', [bw, tar], globalMinMax=False, isLatex=False)
for r in ret:
if ret[0][2].name == 'bw':
bw_sk_rank = ret[0][0]
target_sk_rank = ret[1][0]
else:
bw_sk_rank = ret[1][0]
target_sk_rank = ret[0][0]
t.append(np.median(tar[1:]) - np.median(bw[1:]))
print t[-1]
if bw_sk_rank < target_sk_rank:
aux.append('better')
elif bw_sk_rank == target_sk_rank:
aux.append('same')
else:
for can1 in frontier:
for can2 in frontier:
if can1 == can2:
continue
if model.type1(can1, can2):
frontier.remove(can2)
hv = (total_gen * population_size - len(frontier)) / (total_gen * population_size)
return hv
if __name__ == '__main__':
era_collection = []
for model, text in zip([DTLZ1, DTLZ3, DTLZ5, DTLZ7], ["DTLZ1", "DTLZ3", "DTLZ5", "DTLZ7"]):
for decs in [10, 20, 40]:
for objs in [2, 4, 6, 8]:
print "Now performing for " + text + " " + str(decs) + " " + str(objs)
cur_model = model(decs, objs)
era_val = []
era_val.append(genetic_algorithm(cur_model, differential_evolution(cur_model), False))
# era_val = [cur_model.normalize_val(cur_model.eval(val)) for val in genetic_algorithm(cur_model, differential_evolution(cur_model), True)]
era_val.insert(0, text + "_" + str(decs) + "_" + str(objs) + "_tuned")
era_collection.append(era_val)
era_val = []
era_val.append(genetic_algorithm(cur_model, None, True))
# era_val = [cur_model.normalize_val(cur_model.eval(val)) for val in genetic_algorithm(cur_model, None, True)]
era_val.insert(0, text + "_" + str(decs) + "_" + str(objs) + "_untuned")
era_collection.append(era_val)
print era_collection
print rdivDemo(era_collection)
def performance(f):
l = {}
l_early = {}
l_late = {}
for i in untuned.keys():
if "d" in i and "t" in i:
l["DT"] = untuned[i][0]
l_early["DT"] = early[i][0]
l_late["DT"] = late[i][0]
elif "svm" in i:
l["SVM"] = untuned[i][0]
l_early["SVM"] = early[i][0]
l_late["SVM"] = late[i][0]
elif "rf" in i:
l["RF"] = untuned[i][0]
l_early["RF"] = early[i][0]
l_late["RF"] = late[i][0]
l = OrderedDict(sorted(l.items(), key=lambda t: t[0]))
l_early = OrderedDict(sorted(l_early.items(), key=lambda t: t[0]))
l_late = OrderedDict(sorted(l_late.items(), key=lambda t: t[0]))
temp = []
temp1 = []
temp2 = []
temp3 = OrderedDict()
for i in l.keys():
temp.append(np.median(l[i]))
temp1.append(np.median(l_early[i]))
temp2.append(np.median(l_late[i]))
temp3[i] = []
temp3[i].append(["untuned"] + l[i])
temp3[i].append(["early"] + l_early[i])
temp3[i].append(["late"] + l_late[i])
font = {'size': 80}
plt.rc('font', **font)
paras = {
'lines.linewidth': 8,
'legend.fontsize': 50,
'axes.labelsize': 100,
'legend.frameon': False,
'figure.autolayout': True,
'axes.linewidth': 10
}
plt.rcParams.update(paras)
fig = plt.figure(figsize=(30, 20))
plt.plot(temp, color="green", label='Untuned')
plt.plot(temp1, color="red", label='Early Tuned')
plt.plot(temp2, color="blue", label='Late Tuned')
for i, j in enumerate(l.keys()):
temp_dic = OrderedDict()
ranks = rdivDemo(temp3[j])
for _, _, z in ranks:
temp_dic[z.name] = z.rank + 1
plt.annotate("rk: " + str(temp_dic["untuned"]),
xy=(i, temp[i]),
size=50)
plt.annotate("rk: " + str(temp_dic["early"]),
xy=(i, temp1[i]),
size=50)
plt.annotate("rk: " + str(temp_dic["late"]), xy=(i, temp2[i]), size=50)
plt.xticks(xrange(len(l.keys())), l.keys())
plt.ylabel("Performance", labelpad=30)
plt.xlabel("Learners", labelpad=30)
plt.legend(bbox_to_anchor=(0.9, 1.1), loc=1, ncol=3, borderaxespad=0.1)
plt.savefig("../results/performance_" + f + ".png")
plt.close(fig)
def list2sk(self, lst):
print("\section*{}")
rdivDemo(lst, isLatex=True)
from sk import rdivDemo
import os
folder = "./Results_10-100/"
results = [folder+f for f in os.listdir(folder) if '.csv' in f]
for result in results:
content = open(result).readlines()
content = content[1:]
l = []
for c in content:
c = c.split(',')
l.append([c[0]] + map(float, c[1:]))
family_name = result.split('/')[-1].replace('.csv', '')
rdivDemo(family_name, l, globalMinMax=False, isLatex=True)
def list2sk(self, lst):
print("\subsection*{" + self.projName + "}")
rdivDemo(lst, isLatex=True)
# print ("%3d : %0.7f || %s" % (k, eb, print_string))
if k > max_repeats-1:
return get_era_energy_list(previous_era)
# model = DTLZ7(10,2)
# def dtlz7():
# try:
era_list = []
model_name = ["SA", "MWS", "DE"]
final_era_energy_list = []
i = 0
for optimizer in [sa, mws, de]:
k = 0
for _ in range(20):
model = DTLZ7(10,2)
final_era_energy_list = optimizer(DTLZ7)
final_era_energy_list.insert(0, model_name[i] + str(k))
era_list.append(final_era_energy_list)
k = k+1
i += 1
print sk.rdivDemo(era_list)
# return True
# except:
# dtlz7()
# dtlz7()
from sk import rdivDemo
import pickle
al_eval_p = "../PickleLocker/ActiveLearning_Rank_Diff.p"
mmre_eval_p = "../PickleLocker/Progressive_MMRE_Rank_Diff.p"
rank_eval_p = "../PickleLocker/Progressive_Rank_Rank_Diff.p"
al_evals = pickle.load(open(al_eval_p, "r"))
mmre_evals = pickle.load(open(mmre_eval_p, "r"))
rank_evals = pickle.load(open(rank_eval_p, "r"))
files = sorted(al_evals.keys())
for i, file in enumerate(files):
lists = list()
lists.append(["AL-Rank"] + al_evals[file])
lists.append(["P-Rank"] + rank_evals[file])
lists.append(["P-MMRE"] + mmre_evals[file])
rdivDemo("SS" + str(i + 1), lists, globalMinMax=False, isLatex=True)
def main():
dir = '../Data'
from os import walk
dataName = [Name for _, Name, __ in walk(dir)][0]
numData = len(dataName) # Number of data
Prd = [CART] # , rforest] # , adaboost, logit, knn]
_smoteit = [True] # , False]
_tuneit = [False]
cd = {}
abcd = []
res = {}
for n in xrange(numData):
out11 = []
outA1 = []
out1 = []
outFar = []
outNear = []
outa = []
one, two = explore(dir)
data = [one[i] + two[i] for i in xrange(len(one))]
print('##', dataName[n])
for p in Prd:
train = [dat[0] for dat in withinClass(data[n])]
test = [dat[1] for dat in withinClass(data[n])]
reps = 10
abcd = [[], []]
for t in _tuneit:
tunedParams = None if not t else params
print('### Tuning') if t else print('### No Tuning')
for _smote in _smoteit:
# for _n in xrange(0):
_n = -1
# Training data
for _ in xrange(reps):
train_DF = createTbl(train[_n], isBin=True)
# set_trace()
# Testing data
test_df = createTbl(test[_n], isBin=True)
predRows = []
# Tune?
actual = Bugs(test_df)
before = p(train_DF, test_df,
tunings=tunedParams,
smoteit=True)
tunedParams = None if not t else tuner(p, train[_n])
for predicted, row in zip(before, test_df._rows):
tmp = row.cells
tmp[-2] = predicted
if predicted > 0:
predRows.append(tmp)
predTest = clone(test_df, rows=predRows)
# Find and apply contrast sets
# newTab = treatments(train = train[_n],
# test = test[_n],
# verbose = False,
# smoteit = False).main()
newTab_near = treatments2(train=train[_n], far=False, test=test[_n] # ).main()
, test_df=predTest).main() \
if predRows \
else treatments2(train=train[_n], test=test[_n]).main()
newTab_far = treatments2(train=train[_n], test=test[_n] # ).main()
, test_df=predTest).main() \
if predRows \
else treatments2(train=train[_n], test=test[_n]).main()
after_far = p(train_DF, newTab_far,
tunings=tunedParams,
smoteit=True)
after_near = p(train_DF, newTab_near,
tunings=tunedParams,
smoteit=True)
# print(showoff(dataName[n], before, after))
outa.append(_Abcd(before=actual, after=before))
# set_trace()
cliffsFar = cliffsdelta(Bugs(predTest), after_far)
cliffsNear = cliffsdelta(Bugs(predTest), after_near)
# print(cliffsDelta(Bugs(predTest), after))
# print('Gain = %1.2f' % float(\
# (sum(Bugs(predTest)) - sum(after)) / sum(Bugs(predTest)) * 100), r'%')
outFar.append(cliffsFar)
outNear.append(cliffsNear)
# out1.append(float((sum(before) - sum(after)) / sum(before) * 100))
# out1 = [o for o in out1 if np.isfinite(o)]
outNear.insert(0, dataName[n] + '_Far')
outFar.insert(0, dataName[n] + '_Near')
outa.insert(0, dataName[n])
out11.extend([outNear, outFar])
outA1.append(outa)
try:
print('```')
rdivDemo(out11, isLatex=False)
# rdivDemo(outA1, isLatex = False)
print('```')
except IndexError:
pass
################
ID1 = (model.__name__+' untuned ')#+str(decisions)+'-Decisions '+str(objectives)+'-Objectives')
ID2 = (model.__name__+' tuned ')#+str(decisions)+'-Decisions '+str(objectives)+'-Objectives')
rdivInput[rdivIndex].append(ID1)
for each in HV:
rdivInput[rdivIndex].append(each)
rdivIndex += 1
rdivInput[rdivIndex].append(ID2)
for each in tunedHV:
rdivInput[rdivIndex].append(each)
rdivIndex += 1
ranks = rdivDemo(rdivInput)
'''
print "\nranks",ranks
print ""
print ranks[0][0]
print ranks[1][0]
print ranks[0][1]
print ranks[1][1]
print ""
'''
fo.write('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n')
if ranks[0][1] >= ranks[1][1] :
print "No improvement"
fo.write('No improvement\n')
else :
for d in self.model.decisions():
x, y, z = two.have[d], three.have[d], four.have[d]
if random() < cf:
changed = True
new = x + f * (y - z)
out.have[d] = max(0.0, min(new, 1.0)) # keep in range
if not changed:
d = self.a(self.model.decisions())
out.have[d] = two.have[d]
out.score = self.model.energy(out.have) # remember to score it
out.energy = self.model.energy(out.have)
return out
model = DTLZ7(10, 2)
eras = []
for i in xrange(20):
for algorithm in ["MaxWalkSat", "SA", "DE"]:
if algorithm == "MaxWalkSat":
algo = MaxWalkSat("DTLZ7")
elif algorithm == "SA":
algo = SA("DTLZ7")
elif algorithm == "DE":
algo = DE("DTLZ7")
era = [model.normalize_val(model.eneergy(val)) for val in algo.run()]
era.insert(0, algorithm + str(i))
eras.append(era)
print era
print rdivDemo(eras)
random.seed(DifferentSeeds)
for Algorithm in [sa, mws, de]:
print "ITERATION:", j
if Algorithm.__name__ == "SimulatedAnnealing":
i = 0
if Algorithm.__name__ == "MaxWalkSat":
i = 1
if Algorithm.__name__ == "DifferentialEvolution":
i = 2
solution, enegry = Algorithm(model)
print "------------------------------------------"
print "Model : ", model.__name__
print "Algorithm : ", Algorithm.__name__
print "------------------------------------------"
print "Best solutions: "
print "~~~~~~~~~~~~~~~~~~~~~"
n = 1
for x in solution:
print "%02d" % n, ":", x
n += 1
print "~~~~~~~~~~~~~~~~~~~~~"
print "Energy Evaluation: "
print "~~~~~~~~~~~~~~~~~~~~~"
print enegry
print "~~~~~~~~~~~~~~~~~~~~~"
rdivInput[i].append(enegry)
rdivDemo(rdivInput)
def type3():
eras = 5
era_length = 100
mw_maxtries = 500
mw_maxchanges = 100
mw_p = 0.5
mw_threshold = 2
mw_steps = 10
de_cr = 0.4
de_f = 0.5
kmax = 6000
de_eras = ["DE"]
mws_eras = ["MWS"]
sa_eras = ["SA"]
et_counter = {'SA':0,'MWS':0,'DE':0}
for i in xrange(20):
seed = randomassign()
mw_ea,mw_sol, mw_era = max_walk_sat(mw_maxtries,mw_maxchanges,999,mw_p,mw_steps,seed,eras,era_length)
label = "MWS"+str(i+1)
# +"best: "+str(round(dtlz7(mw_sol),2))
res = [dtlz7(x) for x in mw_era]
res.insert(0,label)
era_collection.append(res)
if mw_ea == True:
et_counter['MWS']+=1
mw_era= []
s_ea,sa_sol, sa_era = simmulated_annealing(kmax,seed,eras,era_length,)
label = "SA"+str(i+1)
# +"best: "+str(round(dtlz7(sa_sol),2))
res = [dtlz7(x) for x in sa_era]
res.insert(0,label)
era_collection.append(res)
if s_ea == True:
et_counter['SA']+=1
sa_era = []
de_ea,de_sol, de_era = differential_evolution( de_cr, de_f,12000,seed,eras,era_length)
label = "DE"+str(i+1)
# +str(round(dtlz7(de_sol),2))
res = [dtlz7(x) for x in de_era]
res.insert(0,label)
era_collection.append(res)
if de_ea == True:
et_counter['DE']+=1
de_era=[]
# mw_ea,mw_sol, mw_era = max_walk_sat(mw_maxtries,mw_maxchanges,999,mw_p,mw_steps,seed,eras,era_length)
# # label = "MWS"+str(i+1)
# # +"best: "+str(round(dtlz7(mw_sol),2))
# res = [dtlz7(x) for x in mw_era]
# # res.insert(0,label)
# mws_eras.extend(res)
# if mw_ea == True:
# et_counter['MWS']+=1
# mw_era= []
#
# s_ea,sa_sol, sa_era = simmulated_annealing(kmax,seed,eras,era_length,)
# # label = "SA"+str(i+1)
# # +"best: "+str(round(dtlz7(sa_sol),2))
# res = [dtlz7(x) for x in sa_era]
# # res.insert(0,label)
# sa_eras.extend(res)
# if s_ea == True:
# et_counter['SA']+=1
# sa_era = []
#
# de_ea,de_sol, de_era = differential_evolution( de_cr, de_f,kmax,seed,eras,era_length)
# # label = "DE"+str(i+1)
# # +str(round(dtlz7(de_sol),2))
# res = [dtlz7(x) for x in de_era]
# # res.insert(0,label)
# de_eras.extend(res)
# if de_ea == True:
# et_counter['DE']+=1
# de_era=[]
#
# era_collection.append(mws_eras)
# era_collection.append(sa_eras)
# era_collection.append(de_eras)
rdivDemo(era_collection)
for key,val in et_counter.items():
print key,val
def _test(name='Apache'):
for name in ['Apache', 'BDBC', 'BDBJ', 'LLVM', 'X264', 'SQL']:
print('## %s \n```' % (name))
R = fileHandler().main(name, reps=10)
rdivDemo(R, isLatex=False)
print('```')
def type3(data):
rdivDemo(data)
def list2sk(self, lst):
print("\section*{}")
rdivDemo(lst, isLatex=True)
def list2sk(self, lst):
# print("\subsection*{" + self.projName + "}")
rdivDemo(lst, isLatex=False)
tune_dict[j] = waste
mode[mode_name[i]] = tune_dict
optimi[x] = mode
for y in mode_name:
tmp_gen = []
tmp_eta = []
for x in optimizers:
for k in gen:
tmp_gen.append([y + "_" + x + "_" + "gen" + "_" + str(k)] +
optimi[x][y]["gen"][k])
for j in eta_n:
tmp_eta.append([y + "_" + x + "_" + "eta" + "_" + str(j)] +
optimi[x][y]["eta"][j])
print(rdivDemo(tmp_gen))
print(rdivDemo(tmp_eta))
#This code has been referenced from https://github.com/ai-se/Spread-HyperVolume/tree/master/HyperVolume
HyperVolume_wrapper()
optimi = {}
for x in optimizers:
mode = {}
for i in range(2):
# print("\n\nModel ",i,"\n\n")
if x == "nsga2":
mode[mode_name[i]] = (NSGA2(model=models[i],
NDIM=NDIM[i],
GEN=20))
elif x == "spea2":
elif Model == POM3:
out += [Model().simulate(elem[:-2])[0]]
return out
def learner(mdl=XOMO, lst=[], reps=24):
train, test = mdl().genData(N=1000)
before = array(predictor(Model=mdl, tbl=train))
for ext in [0, 0.25, 0.5, 0.75]:
for info, prune in zip([0.25, 0.5, 0.75, 1.00], [True, True, True, False]):
prefix = " Base" if ext == 0 else "F=%0.2f, B=%0.2f" % (ext, info)
# print(prefix)
E = [prefix]
for _ in xrange(reps):
newTab = xtrees(train=train, test=test, verbose=False).main()
after = array(predictor(tbl=newTab))
E.append(sum(after) / sum(before))
lst.append(E)
return lst
if __name__ == "__main__":
random.seed(0)
lst = learner(reps=10)
try:
rdivDemo(lst, isLatex=True)
except:
set_trace()
# ------- Debug --------
set_trace()
