def get_data_from_ipet(self): """ Import data from IPET. Create ipet.experiment, add files and execute ipet.collectData. Returns ------- ipet.testrun object """ try: c = Experiment() # Metafiles will be loaded automatically if they are placed next to outfiles c.addOutputFile(self.files[".out"]) if self.files[".err"] is not None: c.addOutputFile(self.files[".err"]) if self.files[".set"] is not None: c.addOutputFile(self.files[".set"]) if self.files[".solu"] is None: msg = "No solu file found." path = ALL_SOLU if os.path.isfile(path): msg = "Adding SoluFile." self.files[".solu"] = path else: msg = "Adding SoluFile." self.logger.info(msg) if self.files[".solu"] is not None: c.addSoluFile(self.files[".solu"]) path = ADD_READERS if os.path.isfile(path): for r in loader.loadAdditionalReaders([path]): c.readermanager.registerReader(r) self.logger.info("Added additional reader: " + r.getName()) for solver in loader.loadAdditionalSolvers(): c.readermanager.addSolver(solver) self.logger.info("Added additional solver: " + solver.getName()) c.collectData() except Exception: traceback.print_exc() msg = "Some kind of ipet error. Aborting..." self._log_failure(msg) raise testruns = c.getTestRuns() if len(testruns) != 1: msg = "Unexpected number of testruns. Expected 1, got: {}".format( len(testruns)) self._log_failure(msg) raise Exception(msg) return testruns[0]
class ExperimentTest(unittest.TestCase): # datasamples = [ # ("meanvarx", 'Datetime_Start', convertTimeStamp(1406905030)), # ("lseu", 'Settings', 'testmode'), # ("misc03", "Datetime_End", convertTimeStamp(1406904997)), # ("findRoot", "Nodes", 8), # ("linking", "LineNumbers_BeginLogFile", 4), # ("j301_2", "LineNumbers_BeginLogFile", 276), # ("j301_2", "LineNumbers_EndLogFile", 575), # ] datasamples = [ # (26, 'Datetime_Start', convertTimeStamp(1406905030)), # (12, 'Settings', 'testmode'), # (14, "Datetime_End", convertTimeStamp(1406904997)), (39, "Nodes", 8), (0, "LineNumbers_BeginLogFile", 4), (1, "LineNumbers_BeginLogFile", 276), (1, "LineNumbers_EndLogFile", 575), ] checkColumns = ['SolvingTime', 'Nodes', 'Datetime_Start', 'GitHash'] def setUp(self): try: os.mkdir(TMPDIR) except FileExistsError: pass self.experiment = Experiment() def tearDown(self): shutil.rmtree(TMPDIR) def test_datacollection(self): fname = "check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out" out_file = os.path.join(DATADIR, fname) solu_file = os.path.join(DATADIR, "short.solu") self.experiment.addOutputFile(out_file) self.experiment.addSoluFile(solu_file) self.experiment.collectData() df = self.experiment.getTestRuns()[0].getData() for index, column, value in self.datasamples: entry = df.loc[index, column] msg = "Wrong value parsed for problem %s in column %s: should be %s, have %s" % \ (index, column, repr(value), repr(entry)) self.assertEqual(entry, value, msg) def test_datacollection_from_stdin(self): fname = "bell3a.out" out_file = os.path.join(DATADIR, fname) with open(out_file, "r") as f: sys.stdin = f self.experiment.addStdinput() self.experiment.collectData() sys.stdin = sys.__stdin__ experimentFromFile = Experiment() experimentFromFile.addOutputFile(out_file) experimentFromFile.collectData() trstdin = self.experiment.getTestRuns()[0] trfile = experimentFromFile.getTestRuns()[0] columns = ["PrimalBound", "DualBound", "SolvingTime"] self.checkTestrunsEqual(trstdin, trfile, columns) def test_problem_name_parsing(self): fname = "check.IP_0s_1s.scip-3.2.1.2.linux.x86_64.gnu.dbg.cpx.opt-low.default.out" out_file = os.path.join(DATADIR, fname) solu_file = os.path.join(DATADIR, "IP_0s_1s.solu") self.experiment.addOutputFile(out_file) self.experiment.addSoluFile(solu_file) self.experiment.collectData() data = self.experiment.getTestRuns()[0].getData() # ensure that the correct number of problems are properly parsed self.assertEqual(len(data), 411) def checkTestrunsEqual(self, tr, tr2, columns=checkColumns): msg = "Testruns do not have exactly same column data." return self.assertIsNone( assert_frame_equal(tr.getData()[columns], tr2.getData()[columns]), msg) def test_saveAndLoad(self): fname = "check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out" out_file = os.path.join(DATADIR, fname) solu_file = os.path.join(DATADIR, "short.solu") self.experiment.addOutputFile(out_file) self.experiment.addSoluFile(solu_file) self.experiment.collectData() save_file = os.path.join(TMPDIR, ".testcomp.cmp") self.experiment.saveToFile(save_file) Experiment.loadFromFile(save_file) tr = self.experiment.getTestRuns()[0] trn_file = os.path.join(DATADIR, ".testrun.trn") tr.saveToFile(trn_file) tr2 = TestRun.loadFromFile(trn_file) self.checkTestrunsEqual(tr, tr2) def test_problemNameRecognition(self): rm = ReaderManager() problemnames2line = {} fname = os.path.join(DATADIR, "problemnames.txt") with open(fname, "r") as problemNames: for line in problemNames: parsedName = rm.getProblemName(line) self.assertIsNotNone( parsedName, "'%s' problem name line was parsed to None" % line) msg = "Error in line '%s'\n: '%s' already contained in line '%s'" % \ (line, parsedName, problemnames2line.get(parsedName)) self.assertTrue((parsedName not in problemnames2line), msg) problemnames2line[parsedName] = line[:] def test_line_numbers(self): fname = "check.MMM.scip-hashing.linux.x86_64.gnu.dbg.cpx.mip-dbg.heuraggr.out" out_file = os.path.join(DATADIR, fname) solu_file = os.path.join(DATADIR, "MMM.solu") self.experiment.addOutputFile(out_file) self.experiment.addSoluFile(solu_file) self.experiment.collectData() data = self.experiment.getTestRuns()[0].data def test_trnfileextension(self): trn_file = os.path.join(DATADIR, ".testrun.trn") self.experiment.addOutputFile(trn_file) self.experiment.collectData() def test_fileExtensions(self): """ Test if an experiment accepts """ # all possible extensions should be accepted for extension in ReaderManager().getFileExtensions(): self.experiment.addOutputFile("bla" + extension) # if called with an unknown extension, this should raise a ValueError for otherextension in [".res", ".txt"]: with self.assertRaises(ValueError): self.experiment.addOutputFile("bla" + otherextension) def test_ListReader(self): lr = ListReader("([ab]c) +([^ ]*)", "testlr") lines = ("ac 1", "bc 2", "ab 3") correctanswers = (("ac", 1), ("bc", 2), None) for idx, line in enumerate(lines): lrlinedata = lr.getLineData(line) self.assertEqual(lrlinedata, correctanswers[idx], "Wrongly parsed line '%s'" % line) def test_parsingOfSettingsFile(self): fname = "check.bugs.scip-221aa62.linux.x86_64.gnu.opt.spx.opt97.default.set" set_file = os.path.join(DATADIR, fname) self.experiment.addOutputFile(set_file) self.experiment.collectData() tr = self.experiment.getTestRuns()[0] values, defaultvalues = tr.getParameterData() crappysettings = collect_settings(set_file) valuesamples = { "constraints/quadratic/scaling": True, "conflict/bounddisjunction/continuousfrac": 0.4, "constraints/soc/sparsifymaxloss": 0.2, "separating/cgmip/nodelimit": 10000, "presolving/abortfac": 0.0001, "vbc/filename": "\"-\"", } for key, val in valuesamples.items(): msg = "wrong parameter value %s parsed for parameter <%s>, should be %s" % \ (repr(values[key]), key, repr(val)) self.assertEqual(val, values[key], msg) msg = "wrong default value %s parsed for parameter <%s>, should be %s" % \ (repr(defaultvalues[key]), key, repr(val)) self.assertEqual(val, defaultvalues[key], msg) for key, val in crappysettings.items(): msg = "wrong parameter value %s parsed for parameter <%s>, should be %s" % \ (repr(values[key]), key, repr(val)) self.assertEqual(val, values[key], msg) def testStatusComparisons(self): goodStatusList = [ Key.ProblemStatusCodes.Ok, Key.ProblemStatusCodes.Better, Key.ProblemStatusCodes.SolvedNotVerified ] badStatusList = [ Key.ProblemStatusCodes.FailAbort, Key.ProblemStatusCodes.FailObjectiveValue, Key.ProblemStatusCodes.Fail ] msg = "Returned status {0} is not the expected {1}" for status, expected in [ (Key.ProblemStatusCodes.getBestStatus(goodStatusList + badStatusList), Key.ProblemStatusCodes.Ok), (Key.ProblemStatusCodes.getBestStatus(*(goodStatusList + badStatusList)), Key.ProblemStatusCodes.Ok), (Key.ProblemStatusCodes.getWorstStatus(goodStatusList + badStatusList), Key.ProblemStatusCodes.FailAbort), (Key.ProblemStatusCodes.getBestStatus(badStatusList + ["dummy"]), "dummy"), (Key.ProblemStatusCodes.getWorstStatus(goodStatusList + ["timelimit"]), "timelimit") ]: self.assertEqual(status, expected, msg.format(status, expected))
return IPETFilterGroup.processXMLElem(tree) @staticmethod def fromXMLFile(xmlfilename): """ parse a file containing an xml string matching the filter group XML representation syntax """ tree = ElementTree.parse(xmlfilename) return IPETFilterGroup.processXMLElem(tree.getroot()) if __name__ == '__main__': comp = Experiment(files=[ '../test/check.short.scip-3.1.0.1.linux.x86_64.gnu.dbg.spx.opt85.testmode.out' ]) comp.addSoluFile('../test/short.solu') comp.collectData() operator = 'ge' expression1 = 'Nodes' expression2 = '2' filter1 = IPETFilter(expression1, expression2, operator, anytestrun="all") filter2 = IPETFilter(expression1, expression2, operator, anytestrun="one") print(filter1.getName()) print(len(comp.getProblemNames())) print( len( filter1.getFilteredList( comp.getProblemNames(), comp.getManager('testrun').getManageables()))) print( len(