def test_saveAndLoad(self):
fname = "check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out"
out_file = os.path.join(DATADIR, fname)
solu_file = os.path.join(DATADIR, "short.solu")
self.experiment.addOutputFile(out_file)
self.experiment.addSoluFile(solu_file)
self.experiment.collectData()
save_file = os.path.join(TMPDIR, ".testcomp.cmp")
self.experiment.saveToFile(save_file)
Experiment.loadFromFile(save_file)
tr = self.experiment.getTestRuns()[0]
trn_file = os.path.join(DATADIR, ".testrun.trn")
tr.saveToFile(trn_file)
tr2 = TestRun.loadFromFile(trn_file)
self.checkTestrunsEqual(tr, tr2)
def test_evalformat(self):
ex = Experiment()
ex.addOutputFile(test_out)
ex.collectData()
for (index, indexsplit) in ((index, indexsplit) for index in eval_index
for indexsplit in eval_indexsplit):
for (cols, fgs) in ((cols, fgs) for cols in self.test_cols
for fgs in self.test_fgs):
ev = IPETEvaluation(index=index, indexsplit=indexsplit)
if cols is not None:
for (ind_c, ind_a) in cols:
col = IPETEvaluationColumn(**columns[ind_c])
if col_filters[ind_c] is not None:
colfilter = IPETFilter(**col_filters[ind_c])
col.addFilter(colfilter)
if ind_a:
agg = Aggregation(**aggs[ind_c])
col.addAggregation(agg)
ev.addColumn(col)
if fgs is not None:
for ind in fgs:
fg = IPETFilterGroup(**filtergroups[ind])
if filters[ind] is not None:
fgfilter = IPETFilter(**filters[ind])
fg.addFilter(fgfilter)
ev.addFilterGroup(fg)
try:
tab_long, tab_agg = ev.evaluate(ex)
except AttributeError as e:
self.assertTrue((
cols is None or fgs is None
), "Either the number of columns or the number of filtergroups should be 0."
)
continue
self.assertEqual(type(tab_long), pd.DataFrame,
"Type of long table wrong.")
self.assertEqual(type(tab_agg), pd.DataFrame,
"Type of aggregated table wrong.")
# do not allow rowindex to be empty
if indexsplit == 0:
indexsplit = 1
rowindex_level = len(index.split()[:indexsplit])
columns_level = len(index.split()[indexsplit:])
self.assertEqual(tab_long.columns.nlevels, columns_level + 1,
"Level of columnindex of long table wrong.")
self.assertEqual(tab_agg.index.nlevels, columns_level + 1,
"Level of rowindex of agg table wrong.")
self.assertEqual(tab_long.index.nlevels, rowindex_level,
"Level of rowindex of long table wrong.")
def test_datacollection_from_stdin(self):
fname = "bell3a.out"
out_file = os.path.join(DATADIR, fname)
with open(out_file, "r") as f:
sys.stdin = f
self.experiment.addStdinput()
self.experiment.collectData()
sys.stdin = sys.__stdin__
experimentFromFile = Experiment()
experimentFromFile.addOutputFile(out_file)
experimentFromFile.collectData()
trstdin = self.experiment.getTestRuns()[0]
trfile = experimentFromFile.getTestRuns()[0]
columns = ["PrimalBound", "DualBound", "SolvingTime"]
self.checkTestrunsEqual(trstdin, trfile, columns)
def resetExperiment():
global _experiment
_experiment = Experiment()
def setUp(self):
try:
os.mkdir(TMPDIR)
except FileExistsError:
pass
self.experiment = Experiment()
class ExperimentTest(unittest.TestCase):
# datasamples = [
# ("meanvarx", 'Datetime_Start', convertTimeStamp(1406905030)),
# ("lseu", 'Settings', 'testmode'),
# ("misc03", "Datetime_End", convertTimeStamp(1406904997)),
# ("findRoot", "Nodes", 8),
# ("linking", "LineNumbers_BeginLogFile", 4),
# ("j301_2", "LineNumbers_BeginLogFile", 276),
# ("j301_2", "LineNumbers_EndLogFile", 575),
# ]
datasamples = [
# (26, 'Datetime_Start', convertTimeStamp(1406905030)),
# (12, 'Settings', 'testmode'),
# (14, "Datetime_End", convertTimeStamp(1406904997)),
(39, "Nodes", 8),
(0, "LineNumbers_BeginLogFile", 4),
(1, "LineNumbers_BeginLogFile", 276),
(1, "LineNumbers_EndLogFile", 575),
]
checkColumns = ['SolvingTime', 'Nodes', 'Datetime_Start', 'GitHash']
def setUp(self):
try:
os.mkdir(TMPDIR)
except FileExistsError:
pass
self.experiment = Experiment()
def tearDown(self):
shutil.rmtree(TMPDIR)
def test_datacollection(self):
fname = "check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out"
out_file = os.path.join(DATADIR, fname)
solu_file = os.path.join(DATADIR, "short.solu")
self.experiment.addOutputFile(out_file)
self.experiment.addSoluFile(solu_file)
self.experiment.collectData()
df = self.experiment.getTestRuns()[0].getData()
for index, column, value in self.datasamples:
entry = df.loc[index, column]
msg = "Wrong value parsed for problem %s in column %s: should be %s, have %s" % \
(index, column, repr(value), repr(entry))
self.assertEqual(entry, value, msg)
def test_datacollection_from_stdin(self):
fname = "bell3a.out"
out_file = os.path.join(DATADIR, fname)
with open(out_file, "r") as f:
sys.stdin = f
self.experiment.addStdinput()
self.experiment.collectData()
sys.stdin = sys.__stdin__
experimentFromFile = Experiment()
experimentFromFile.addOutputFile(out_file)
experimentFromFile.collectData()
trstdin = self.experiment.getTestRuns()[0]
trfile = experimentFromFile.getTestRuns()[0]
columns = ["PrimalBound", "DualBound", "SolvingTime"]
self.checkTestrunsEqual(trstdin, trfile, columns)
def test_problem_name_parsing(self):
fname = "check.IP_0s_1s.scip-3.2.1.2.linux.x86_64.gnu.dbg.cpx.opt-low.default.out"
out_file = os.path.join(DATADIR, fname)
solu_file = os.path.join(DATADIR, "IP_0s_1s.solu")
self.experiment.addOutputFile(out_file)
self.experiment.addSoluFile(solu_file)
self.experiment.collectData()
data = self.experiment.getTestRuns()[0].getData()
# ensure that the correct number of problems are properly parsed
self.assertEqual(len(data), 411)
def checkTestrunsEqual(self, tr, tr2, columns=checkColumns):
msg = "Testruns do not have exactly same column data."
return self.assertIsNone(
assert_frame_equal(tr.getData()[columns],
tr2.getData()[columns]), msg)
def test_saveAndLoad(self):
fname = "check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out"
out_file = os.path.join(DATADIR, fname)
solu_file = os.path.join(DATADIR, "short.solu")
self.experiment.addOutputFile(out_file)
self.experiment.addSoluFile(solu_file)
self.experiment.collectData()
save_file = os.path.join(TMPDIR, ".testcomp.cmp")
self.experiment.saveToFile(save_file)
Experiment.loadFromFile(save_file)
tr = self.experiment.getTestRuns()[0]
trn_file = os.path.join(DATADIR, ".testrun.trn")
tr.saveToFile(trn_file)
tr2 = TestRun.loadFromFile(trn_file)
self.checkTestrunsEqual(tr, tr2)
def test_problemNameRecognition(self):
rm = ReaderManager()
problemnames2line = {}
fname = os.path.join(DATADIR, "problemnames.txt")
with open(fname, "r") as problemNames:
for line in problemNames:
parsedName = rm.getProblemName(line)
self.assertIsNotNone(
parsedName,
"'%s' problem name line was parsed to None" % line)
msg = "Error in line '%s'\n: '%s' already contained in line '%s'" % \
(line, parsedName, problemnames2line.get(parsedName))
self.assertTrue((parsedName not in problemnames2line), msg)
problemnames2line[parsedName] = line[:]
def test_line_numbers(self):
fname = "check.MMM.scip-hashing.linux.x86_64.gnu.dbg.cpx.mip-dbg.heuraggr.out"
out_file = os.path.join(DATADIR, fname)
solu_file = os.path.join(DATADIR, "MMM.solu")
self.experiment.addOutputFile(out_file)
self.experiment.addSoluFile(solu_file)
self.experiment.collectData()
data = self.experiment.getTestRuns()[0].data
def test_trnfileextension(self):
trn_file = os.path.join(DATADIR, ".testrun.trn")
self.experiment.addOutputFile(trn_file)
self.experiment.collectData()
def test_fileExtensions(self):
"""
Test if an experiment accepts
"""
# all possible extensions should be accepted
for extension in ReaderManager().getFileExtensions():
self.experiment.addOutputFile("bla" + extension)
# if called with an unknown extension, this should raise a ValueError
for otherextension in [".res", ".txt"]:
with self.assertRaises(ValueError):
self.experiment.addOutputFile("bla" + otherextension)
def test_ListReader(self):
lr = ListReader("([ab]c) +([^ ]*)", "testlr")
lines = ("ac 1", "bc 2", "ab 3")
correctanswers = (("ac", 1), ("bc", 2), None)
for idx, line in enumerate(lines):
lrlinedata = lr.getLineData(line)
self.assertEqual(lrlinedata, correctanswers[idx],
"Wrongly parsed line '%s'" % line)
def test_parsingOfSettingsFile(self):
fname = "check.bugs.scip-221aa62.linux.x86_64.gnu.opt.spx.opt97.default.set"
set_file = os.path.join(DATADIR, fname)
self.experiment.addOutputFile(set_file)
self.experiment.collectData()
tr = self.experiment.getTestRuns()[0]
values, defaultvalues = tr.getParameterData()
crappysettings = collect_settings(set_file)
valuesamples = {
"constraints/quadratic/scaling": True,
"conflict/bounddisjunction/continuousfrac": 0.4,
"constraints/soc/sparsifymaxloss": 0.2,
"separating/cgmip/nodelimit": 10000,
"presolving/abortfac": 0.0001,
"vbc/filename": "\"-\"",
}
for key, val in valuesamples.items():
msg = "wrong parameter value %s parsed for parameter <%s>, should be %s" % \
(repr(values[key]), key, repr(val))
self.assertEqual(val, values[key], msg)
msg = "wrong default value %s parsed for parameter <%s>, should be %s" % \
(repr(defaultvalues[key]), key, repr(val))
self.assertEqual(val, defaultvalues[key], msg)
for key, val in crappysettings.items():
msg = "wrong parameter value %s parsed for parameter <%s>, should be %s" % \
(repr(values[key]), key, repr(val))
self.assertEqual(val, values[key], msg)
def testStatusComparisons(self):
goodStatusList = [
Key.ProblemStatusCodes.Ok, Key.ProblemStatusCodes.Better,
Key.ProblemStatusCodes.SolvedNotVerified
]
badStatusList = [
Key.ProblemStatusCodes.FailAbort,
Key.ProblemStatusCodes.FailObjectiveValue,
Key.ProblemStatusCodes.Fail
]
msg = "Returned status {0} is not the expected {1}"
for status, expected in [
(Key.ProblemStatusCodes.getBestStatus(goodStatusList +
badStatusList),
Key.ProblemStatusCodes.Ok),
(Key.ProblemStatusCodes.getBestStatus(*(goodStatusList +
badStatusList)),
Key.ProblemStatusCodes.Ok),
(Key.ProblemStatusCodes.getWorstStatus(goodStatusList +
badStatusList),
Key.ProblemStatusCodes.FailAbort),
(Key.ProblemStatusCodes.getBestStatus(badStatusList + ["dummy"]),
"dummy"),
(Key.ProblemStatusCodes.getWorstStatus(goodStatusList +
["timelimit"]), "timelimit")
]:
self.assertEqual(status, expected, msg.format(status, expected))
def get_data_from_ipet(self):
"""
Import data from IPET.
Create ipet.experiment, add files and execute ipet.collectData.
Returns
-------
ipet.testrun object
"""
try:
c = Experiment()
# Metafiles will be loaded automatically if they are placed next to outfiles
c.addOutputFile(self.files[".out"])
if self.files[".err"] is not None:
c.addOutputFile(self.files[".err"])
if self.files[".set"] is not None:
c.addOutputFile(self.files[".set"])
if self.files[".solu"] is None:
msg = "No solu file found."
path = ALL_SOLU
if os.path.isfile(path):
msg = "Adding SoluFile."
self.files[".solu"] = path
else:
msg = "Adding SoluFile."
self.logger.info(msg)
if self.files[".solu"] is not None:
c.addSoluFile(self.files[".solu"])
path = ADD_READERS
if os.path.isfile(path):
for r in loader.loadAdditionalReaders([path]):
c.readermanager.registerReader(r)
self.logger.info("Added additional reader: " + r.getName())
for solver in loader.loadAdditionalSolvers():
c.readermanager.addSolver(solver)
self.logger.info("Added additional solver: " +
solver.getName())
c.collectData()
except Exception:
traceback.print_exc()
msg = "Some kind of ipet error. Aborting..."
self._log_failure(msg)
raise
testruns = c.getTestRuns()
if len(testruns) != 1:
msg = "Unexpected number of testruns. Expected 1, got: {}".format(
len(testruns))
self._log_failure(msg)
raise Exception(msg)
return testruns[0]
"""
tree = ElementTree.fromstring(xmlstring)
return IPETFilterGroup.processXMLElem(tree)
@staticmethod
def fromXMLFile(xmlfilename):
"""
parse a file containing an xml string matching the filter group XML representation syntax
"""
tree = ElementTree.parse(xmlfilename)
return IPETFilterGroup.processXMLElem(tree.getroot())
if __name__ == '__main__':
comp = Experiment(files=[
'../test/check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out'
])
comp.addSoluFile('../test/short.solu')
comp.collectData()
operator = 'ge'
expression1 = 'Nodes'
expression2 = '2'
filter1 = IPETFilter(expression1, expression2, operator, anytestrun="all")
filter2 = IPETFilter(expression1, expression2, operator, anytestrun="one")
print(filter1.getName())
print(len(comp.getProblemNames()))
print(
len(
filter1.getFilteredList(
comp.getProblemNames(),
comp.getManager('testrun').getManageables())))