def comparison_workflow(self, argDict, expected_iapyx_iedb_rewrites_path, expected_eval_path, db_name_append): # get custom save path argDict['data_save_path'] = self.custom_save_path( argDict, db_name_append) # --------------------------------------------------------------- # # testing set up. if os.path.isfile("./IR*.db*"): logging.debug( " COMPARISON WORKFLOW : removing rogue IR*.db* files.") os.remove("./IR*.db*") testDB = "./IR" + db_name_append + ".db" IRDB = sqlite3.connect(testDB) cursor = IRDB.cursor() # --------------------------------------------------------------- # # reset counters for new test dedt.globalCounterReset() # --------------------------------------------------------------- # # runs through function to make sure it finishes with expected error # run translator programData = dedt.translateDedalus(argDict, cursor) # portray actual output program lines as a single string iapyx_results = self.getActualResults(programData) if self.PRINT_STOP: print iapyx_results sys.exit("print stop.") # ========================================================== # # IAPYX COMPARISON # # grab expected output results as a string if self.COMPARE_PROGS: expected_iapyx_results = None with open(expected_iapyx_iedb_rewrites_path, 'r') as expectedFile: expected_iapyx_results = expectedFile.read() self.assertEqual(iapyx_results, expected_iapyx_results) # ========================================================== # # EVALUATION COMPARISON self.evaluate(programData, expected_eval_path, argDict) # --------------------------------------------------------------- # #clean up testing IRDB.close() os.remove(testDB)
def run_workflow(self, test_id, argDict, input_file, eval_type, RUN_EVAL=True): # --------------------------------------------------------------- # # check if dedalus if input_file.endswith(".ded"): IS_DED = True elif input_file.endswith(".olg"): IS_DED = False else: raise Exception( "INVALID FILE TYPE : make sure the input file is either .ded or .olg" ) # --------------------------------------------------------------- # # make sure data dir exists if not os.path.exists(argDict["data_save_path"]): self.make_data_dir(argDict["data_save_path"], test_id) # --------------------------------------------------------------- # if not IS_DED: # extract program string program = self.extract_program_array(input_file) table_list = self.extract_table_list(program) program_data = [program, table_list] # run program with c4 wrapper if RUN_EVAL: eval_results_dict = self.run_program_c4(program_data, argDict) else: eval_results_dict = [] # --------------------------------------------------------------- # else: # --------------------------------------------------------------- # # set up IR data test_db = argDict["data_save_path"] + "/IR_" + test_id + ".db" if os.path.exists(test_db): os.remove(test_db) IRDB = sqlite3.connect(test_db) cursor = IRDB.cursor() dedt.createDedalusIRTables(cursor) dedt.globalCounterReset() # --------------------------------------------------------------- # # run translator # grab only the program lines and table list. program_data = dedt.translateDedalus(argDict, cursor)[0] # run program in c4 if RUN_EVAL: eval_results_dict = self.run_program_c4(program_data, argDict) else: eval_results_dict = [] # --------------------------------------------------------------- # # run analysis analysis_results_dict = self.run_analysis(program_data, eval_results_dict, IS_DED) print print "======================================" print "========== ANALYSIS RESULTS for " + eval_type + " ==========" print for res in analysis_results_dict: print res + " => " + str(analysis_results_dict[res]) print print "======================================" print # --------------------------------------------------------------- # # save analysis data to file self.save_analysis_data_to_file(analysis_results_dict, test_id, argDict) return eval_results_dict
def test_async_1(self): test_id = "pycosat_async_1" test_input_file_name = "async_1_driver" test_db = "./IR_" + test_id + ".db" logging.debug(">> RUNNING TEST '" + test_id + "' <<<") # --------------------------------------------------------------- # # set up test if os.path.exists(test_db): os.remove(test_db) IRDB = sqlite3.connect(test_db) cursor = IRDB.cursor() dedt.createDedalusIRTables(cursor) dedt.globalCounterReset() # --------------------------------------------------------------- # # specify input file paths inputfile = "./dedalus_drivers/" + test_input_file_name + ".ded" # --------------------------------------------------------------- # # get argDict argDict = self.get_arg_dict(inputfile) argDict["nodes"] = ["Node1", "Node2", "Server"] argDict["EOT"] = 3 argDict["EFF"] = 0 argDict[ 'settings'] = "./settings_files/settings_dm_allow_not_clocks.ini" if not os.path.exists(argDict["data_save_path"]): cmd = "mkdir " + argDict["data_save_path"] logging.debug(" TEST " + test_id.upper() + " : running cmd = " + cmd) os.system(cmd) # --------------------------------------------------------------- # # generate orik rgg orik_rgg = self.get_orik_rgg( argDict, \ inputfile, \ cursor, \ test_id ) # --------------------------------------------------------------- # # generate fault hypotheses pycosat_solver = PYCOSAT_Solver.PYCOSAT_Solver(argDict, orik_rgg) logging.debug(" TEST " + test_id.upper() + " : cnf_fmla_list :") for f in pycosat_solver.cnf_fmla_list: logging.debug(f) # get all the solns for all the fmlas for the provenance tree all_solns = self.get_all_solns(pycosat_solver) if self.PRINT_STOP: print "PRINTING ALL SOLNS:" print all_solns sys.exit("hit print stop.") expected_all_solns = [] self.assertEqual(all_solns, expected_all_solns) # --------------------------------------------------------------- # # clean up yo mess if os.path.exists(test_db): os.remove(test_db)
def comparison_workflow( self, \ inputfile, \ expected_iapyx_comb_path, \ test_id, \ nodes=["a", "b", "c"], \ eot=4, \ eff=4 ) : # --------------------------------------------------------------- # # get argDict argDict = self.getArgDict(inputfile, negprov=True, nodes=nodes, eot=eot, eff=eff) origArgDict = self.getArgDict(inputfile, negprov=False, nodes=nodes, eot=eot, eff=eff) argDict["data_save_path"] += test_id origArgDict["data_save_path"] = argDict["data_save_path"] self.make_data_dir(argDict["data_save_path"], test_id) # --------------------------------------------------------------- # # testing set up. testDB = "./" + argDict["data_save_path"] + "/IR_" + test_id testDB_combo = testDB + "_combo.db" testDB_orig = testDB + "_orig.db" if os.path.isfile(testDB_combo): logging.debug(" COMPARISON WORKFLOW : removing rogue IR file.") os.remove(testDB_combo) if os.path.isfile(testDB_orig): logging.debug(" COMPARISON WORKFLOW : removing rogue IR file.") os.remove(testDB_orig) IRDB_combo = sqlite3.connect(testDB_combo) cursor_combo = IRDB_combo.cursor() dedt.globalCounterReset() programData = dedt.translateDedalus(argDict, cursor_combo) # Clear db. IRDB_combo.close() os.remove(testDB_combo) if os.path.isfile(testDB_orig): logging.debug(" COMPARISON WORKFLOW : removing rogue IR file.") os.remove(testDB_orig) IRDB_orig = sqlite3.connect(testDB_orig) cursor_orig = IRDB_orig.cursor() dedt.globalCounterReset() # run the translator with negprov turned off to compare results origProgData = dedt.translateDedalus(origArgDict, cursor_orig) # portray actual output program lines as a single string iapyx_results = self.getActualResults(programData[0]) if self.PRINT_STOP: print iapyx_results sys.exit("PRINT STOP") # ========================================================== # # IAPYX COMPARISON # # grab expected output results as a string expected_iapyx_results = None with open(expected_iapyx_comb_path, 'r') as expectedFile: expected_iapyx_results = expectedFile.read() self.assertEqual(iapyx_results, expected_iapyx_results) # ========================================================== # # EVALUATION COMPARISON self.evaluate(programData, origProgData, argDict) # --------------------------------------------------------------- # #clean up testing IRDB_orig.close() os.remove(testDB_orig)
def test_simplog(self): test_id = "simplog" test_db = "./IR_" + test_id + ".db" logging.debug(">> RUNNING TEST '" + test_id + "' <<<") # --------------------------------------------------------------- # # set up test if os.path.exists(test_db): os.remove(test_db) IRDB = sqlite3.connect(test_db) cursor = IRDB.cursor() dedt.createDedalusIRTables(cursor) dedt.globalCounterReset() # --------------------------------------------------------------- # # specify input file paths inputfile = "./dedalus_drivers/" + test_id + "_driver.ded" # --------------------------------------------------------------- # # get argDict argDict = self.get_arg_dict(inputfile) argDict["nodes"] = ["a", "b", "c"] argDict["EOT"] = 4 argDict["EFF"] = 2 if not os.path.exists(argDict["data_save_path"]): cmd = "mkdir " + argDict["data_save_path"] logging.debug(" TEST SIMPLOG : running cmd = " + cmd) os.system(cmd) # --------------------------------------------------------------- # # generate orik rgg orik_rgg = self.get_orik_rgg( argDict, \ inputfile, \ cursor, \ test_id ) # --------------------------------------------------------------- # # generate fault hypotheses pycosat_solver = PYCOSAT_Solver.PYCOSAT_Solver(argDict, orik_rgg) logging.debug(" SIMPLOG : cnf_fmla_list :") for f in pycosat_solver.cnf_fmla_list: logging.debug(f) # get all the solns for all the fmlas for the provenance tree all_solns = self.get_all_solns(pycosat_solver) if self.PRINT_STOP: print all_solns sys.exit("hit print stop.") expected_all_solns = [["clock(['a','b','1','2'])"], ["clock(['a','c','1','2'])"]] self.assertEqual(all_solns, expected_all_solns) # --------------------------------------------------------------- # # clean up yo mess if os.path.exists(test_db): os.remove(test_db)