def test_xml(self):
"""
test construction of modified evaluations, and if they persist after constructing a twin directly from the XML representation
"""
modificationlist = [{},
{"sortlevel":1},
{"index":"foo bar"},
{"defaultgroup":"bar"}
]
for mod in modificationlist:
evaluation = IPETEvaluation(**mod)
evaluation2 = IPETEvaluation.fromXML(ElementTree.tostring(evaluation.toXMLElem()))
self.assertTrue(evaluation.equals(evaluation2),
"The two evaluations are not the same after modifying %s" % mod)
def test_evalformat(self):
ex = Experiment()
ex.addOutputFile(test_out)
ex.collectData()
for (index, indexsplit) in ((index, indexsplit) for index in eval_index
for indexsplit in eval_indexsplit):
for (cols, fgs) in ((cols, fgs) for cols in self.test_cols
for fgs in self.test_fgs):
ev = IPETEvaluation(index=index, indexsplit=indexsplit)
if cols is not None:
for (ind_c, ind_a) in cols:
col = IPETEvaluationColumn(**columns[ind_c])
if col_filters[ind_c] is not None:
colfilter = IPETFilter(**col_filters[ind_c])
col.addFilter(colfilter)
if ind_a:
agg = Aggregation(**aggs[ind_c])
col.addAggregation(agg)
ev.addColumn(col)
if fgs is not None:
for ind in fgs:
fg = IPETFilterGroup(**filtergroups[ind])
if filters[ind] is not None:
fgfilter = IPETFilter(**filters[ind])
fg.addFilter(fgfilter)
ev.addFilterGroup(fg)
try:
tab_long, tab_agg = ev.evaluate(ex)
except AttributeError as e:
self.assertTrue((
cols is None or fgs is None
), "Either the number of columns or the number of filtergroups should be 0."
)
continue
self.assertEqual(type(tab_long), pd.DataFrame,
"Type of long table wrong.")
self.assertEqual(type(tab_agg), pd.DataFrame,
"Type of aggregated table wrong.")
# do not allow rowindex to be empty
if indexsplit == 0:
indexsplit = 1
rowindex_level = len(index.split()[:indexsplit])
columns_level = len(index.split()[indexsplit:])
self.assertEqual(tab_long.columns.nlevels, columns_level + 1,
"Level of columnindex of long table wrong.")
self.assertEqual(tab_agg.index.nlevels, columns_level + 1,
"Level of rowindex of agg table wrong.")
self.assertEqual(tab_long.index.nlevels, rowindex_level,
"Level of rowindex of long table wrong.")
def setUp(self):
self.ex = HelpExperiment()
ev = IPETEvaluation(defaultgroup="C1",
index="A C"
)
ev.addColumn(IPETEvaluationColumn(origcolname="D"))
fg = IPETFilterGroup("Test")
fi = IPETFilter(operator="keep", datakey="A")
fi.addChild(IPETValue("A1"))
fg.addFilter(fi)
fg2 = IPETFilterGroup("Test2")
fi2 = IPETFilter(operator="keep", datakey="A")
fi2.addChild(IPETValue("A2"))
fg2.addFilter(fi2)
ev.addFilterGroup(fg2)
ev.addFilterGroup(fg)
self.ev = ev
self.fg = fg
def test_xmlWrite(self):
classes = [(IPETFilter, "filter"), (Aggregation, "agg"),
(IPETFilterGroup, "group"), (IPETValue, "instance"),
(IPETEvaluationColumn, "column")]
for cl, basename in classes:
node = cl()
try:
saveAsXML(node, os.path.join(TMPDIR, basename + ".xml"))
except TypeError as e:
raise e
ev = IPETEvaluation.fromXMLFile(EVALTEST)
try:
saveAsXML(ev, os.path.join(TMPDIR, "test.xml"))
except TypeError as e:
print(ev.attributesToStringDict())
raise e
def setUp(self):
"""Set up the evaluation
The evaluation has only one column
for the 'Data' key, but applies
filtering on the 'FilterData' key
"""
self.ex = HelperExperiment()
ev = IPETEvaluation(index="Index", indexsplit="1")
ev.addColumn(IPETEvaluationColumn(origcolname="Data"))
fg = IPETFilterGroup("TestGroup")
fi = IPETFilter(expression1="FilterData",
expression2=-1.0,
operator="eq")
fg.addFilter(fi)
ev.addFilterGroup(fg)
self.ev = ev
def setUp(self):
"""
create an evaluation around a helper experiment
"""
self.ex = HelpExperiment()
ev = IPETEvaluation(defaultgroup="C1",
index="A B C",
fillin=True,
indexsplit=3)
ev.addColumn(
IPETEvaluationColumn(origcolname="D", alternative="100",
maxval=15))
fg = IPETFilterGroup("Test")
fi = IPETFilter(operator="keep", datakey="A")
fi.addChild(IPETValue("A1"))
fg.addFilter(fi)
ev.addFilterGroup(fg)
self.ev = ev
self.fg = fg
def test_testEvaluateXML(self):
ev = IPETEvaluation.fromXMLFile(EVALTEST)
def test_constructor(self):
"""
test if the constructor is working without arguments
"""
evaluation = IPETEvaluation()
def createAndEvaluateColumn(self, col):
ev = IPETEvaluation(index="A B C", indexsplit=3)
ev.addColumn(col)
ev.evaluate(self.helper)
return ev
parent = self
me = IpetTreeViewItem(editable, parent)
self.editable2item[editable] = me
self.bindItemIcon(me, editable)
if editable.getChildren():
for idx, child in enumerate(editable.getChildren()):
self.createAndAddItem(child, me)
self.expandItem(me)
editframecontent = None
if __name__ == "__main__":
ev = IPETEvaluation.fromXMLFile("test/testevaluate.xml")
app = QApplication(sys.argv)
mainwindow = QMainWindow()
thewidget = QWidget(mainwindow)
treeview = IpetTreeView(thewidget)
layout = QHBoxLayout()
layout.addWidget(treeview)
editframe = QFrame(thewidget)
layout.addWidget(editframe)
layout2 = QGridLayout()
editframe.setLayout(layout2)
def get(self, base_id):
"""
Answer to GET requests.
Parameters
----------
parent_id
Rubberband id of parent
Compare TestRuns with IPET
"""
if not base_id:
raise HTTPError(404)
# read testrunids
comparison_ids = self.get_argument("compare", None)
if comparison_ids is not None:
testrunids = list(set(comparison_ids.split(",")))
if base_id in testrunids:
raise HTTPError(404)
# get testruns and default
baserun = get_testruns(base_id)
testruns = []
if comparison_ids is not None:
testruns = get_testruns(testrunids)
# timestamps
basehash = baserun.git_hash
times = {t.git_hash: datetime.strftime(t.git_commit_timestamp, FORMAT_DATE)
for t in testruns + [baserun]}
hashes = set([t.git_hash for t in testruns + [baserun]])
if len(hashes) > 2:
raise HTTPError(404)
hashes.remove(basehash)
comparehash = None
if len(hashes) == 1:
comparehash = hashes.pop()
committime = times[comparehash]
else: # len(hashes) == 0:
committime = times[basehash]
# tolerance
tolerance = float(self.get_argument("tolerance", default=1e-6))
if tolerance == "":
tolerance = 1e-6
# evaluate with ipet
ex, _ = setup_experiment(testruns + [baserun], "")
evalstring = """<?xml version="1.0" ?>
<Evaluation comparecolformat="%.3f" index="ProblemName Settings Seed Permutation GitHash"
indexsplit="-1" fillin="True">
<Column formatstr="%.2f" name="T" origcolname="SolvingTime" minval="0.5"
comp="quot shift. by 1" maxval="TimeLimit" alternative="TimeLimit"
reduction="shmean shift. by 1">
<Aggregation aggregation="shmean" name="sgm" shiftby="1.0"/>
</Column>
<Column formatstr="%.2f" origcolname="TimeLimit" alternative="{tl}"
reduction="mean">
</Column>
<FilterGroup name="all"/>
<FilterGroup name="clean">
<Filter anytestrun="all" expression1="_abort_" expression2="0" operator="eq"/>
<Filter anytestrun="all" expression1="_fail_" expression2="0" operator="eq"/>
</FilterGroup>
</Evaluation>
""".format(tl=baserun.time_limit)
ev = IPETEvaluation.fromXML(evalstring)
ev.set_validate(ALL_SOLU)
ev.set_feastol(tolerance)
set_defaultgroup(ev, ex, base_id)
# do evaluation
longtable, aggtable = ev.evaluate(ex)
# df = aggtable[["_count_","_solved_","T_sgm(1.0)Q","T_sgm(1.0)"]]
if comparehash is not None:
cleanindex = ("clean", comparehash)
allindex = ("all", comparehash)
commithash = comparehash
else:
cleanindex = ("clean", basehash)
allindex = ("all", basehash)
commithash = basehash
allcount = aggtable["_count_"][allindex]
allsolved = aggtable["_solved_"][allindex]
alltime = aggtable["T_sgm(1.0)"][allindex]
cleancount = aggtable["_count_"][cleanindex]
cleansolved = aggtable["_solved_"][cleanindex]
cleantime = aggtable["T_sgm(1.0)"][cleanindex]
self.write(",".join(list(map(str, [
commithash,
committime,
allcount,
allsolved,
alltime,
cleancount,
cleansolved,
cleantime]))))