def benchmarkHadd(problemfile, domainfile, timeout): results = {} task, _, _ = prepareTask(problemfile, domainfile, results) print " searching h^add ...", start = time() h_add_achiever = {} h, plan = run_with_timeout(timeout, None, discover_plan, task, h_add_achiever) serialized_plan = serialize_plan(task, plan) results['plan'] = ", ".join([op.name for op in serialized_plan]) results['plancost'] = sum([op.cost for op in serialized_plan]) results['h_add_time'] = time() - start print "H:%s, T:%d" % (str(results['plancost']), results['h_add_time']) print " optimizing plan ...", start = time() plan = run_with_timeout(timeout, None, opimize_plan, task, plan, h_add_achiever) serialized_plan = serialize_plan(task, plan) results['optimized_plan'] = ", ".join([op.name for op in serialized_plan]) results['optimized_plancost'] = sum([op.cost for op in serialized_plan]) results['optimize_time'] = time() - start print "H:%s, T:%d" % (results['optimized_plancost'], results['optimize_time']) if plan is None: print " Timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) return ProblemResults(problemfile, h_add=h, **results)
def benchmarkHeuristic(problemfile, domainfile, what_to_test, timeout=None): results = {} task, sas_task, translationkey = prepareTask(problemfile, domainfile, results) print " Solving ...", debug_value_list = DebugValueList() start = time() h = run_with_timeout(timeout, None, calculate_lmcut, task, debug_value_list=debug_value_list) solve_time = time() - start if h is None: print " Timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) print "H:%s, T:%d" % (str(h), solve_time) if 'cuts' in what_to_test: results['valid_cut'] = validateCuts(debug_value_list, task, all=True) if 'pcf' in what_to_test: results['valid_pcf'] = validatePcf(debug_value_list, task, all=True, silent=False) if 'relevance' in what_to_test: results['valid_relevance_analysis'] = validateRelevanceAnalysis( sas_task, translationkey, h, timeout=timeout) return ProblemResults(problemfile, heuristic=h, solve_time=solve_time, **results)
def benchmarkSearch(problemfile, domainfile, what_to_test, timeout=None): results = {} task, _, _ = prepareTask(problemfile, domainfile, results) print " Searching for h^+ ...", start = time() search = BranchAndBoundSearch(task, AchieveLandmarksOrGoalsOperatorSelector()) try: h = run_with_timeout(timeout, None, search.run, validateCuts=("cuts" in what_to_test)) except RuntimeError: print "Recusion depth exceeded" return ProblemResults(problemfile, error="Recusion depth exceeded") solve_time = time() - start if h is None: print " Timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) print "H:%s, T:%d" % (str(h), solve_time) results["plan"] = ", ".join(map(str, search.best_plan)) print " With plan:", results["plan"] if "cuts" in what_to_test: # cut validation will assert False if there is any invalid cut. # write in the file to remember that all cuts were validated results["valid_cut"] = True return ProblemResults(problemfile, h_plus=h, solve_time=solve_time, **results)
def benchmarkHmax(problemfile, domainfile, hmax_function, timeout=None): results = {} task, _, _ = prepareTask(problemfile, domainfile, results) print " Solving ...", start = time() h = run_with_timeout(timeout, None, hmax_function, task) solve_time = time() - start if h is None: print " Timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) print "H:%s, T:%d" % (str(h), solve_time) return ProblemResults(problemfile, h_max=h, solve_time=solve_time, **results)
def validateRelevanceAnalysis(sas_task, translationkey, filtered_h, timeout=None, silent=False): print " validating relevance analysis... ", unfiltered_task = delete_relaxation(sas_task, translationkey) unfiltered_task.convert_to_canonical_form() unfiltered_task.crossreference() unfiltered_h = run_with_timeout(timeout, None, calculate_lmcut, unfiltered_task) if unfiltered_h is None: print "timed out" return True if unfiltered_h != filtered_h: if not silent: assert False, "Relevance analysis changed heuristic from %d to %d" % ( unfiltered_h, filtered_h) print "different heuristic value" return False print "same heuristic value" return True
def compareTask(problemfile, domainfile, what_to_compare, timeout=None): print " Translation ...", start = time() old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') task = translate.pddl.open(task_filename=problemfile, domain_filename=domainfile) sas_task, translationkey, _ = pddl_to_sas(task) sys.stdout = old_stdout print time() - start print " Relaxing ...", start = time() task = delete_relaxation(sas_task, translationkey) print time() - start print " Filtering ...", start = time() filter_irrelevant_variables(task) task.convert_to_canonical_form() task.crossreference() print time() - start print " Conversion ...", start = time() malte_task, operator_translator = translate_relaxed_task(task) crossreference_task(malte_task) print time() - start malte_debug_value_list, my_debug_value_list = (DebugValueList(), DebugValueList()) print " Mine", start = time() my_h = run_with_timeout(timeout, None, calculate_lmcut, task, debug_value_list=my_debug_value_list) my_t = time() - start if my_h is None: print " Timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) print " H:%s, T:%d" % (str(my_h), my_t * 1000) print " Malte", start = time() malte_h = run_with_timeout(timeout, None, additive_hmax, malte_task, debug_value_list=malte_debug_value_list, pcfs=(translate_pcf(step.pcf, malte_task, operator_translator) for step in my_debug_value_list.steps)) malte_t = time() - start if malte_h is None: print " Malte timed out" return ProblemResults(problemfile, error="Took longer than %d seconds" % timeout) print " H:%s, T:%d" % (str(malte_h), malte_t * 1000) times = (my_t, malte_t, my_h, malte_h) samehmax, samegoalzone, valid_cut = (None, None, None) if 'hmax' in what_to_compare: samehmax = compareHmax(my_debug_value_list, malte_debug_value_list, silent=True, all=True) if 'goalzone' in what_to_compare: samegoalzone = compareGoalZone(my_debug_value_list, malte_debug_value_list, silent=False, all=True) if 'cuts' in what_to_compare: valid_cut = compareCuts(my_debug_value_list, malte_debug_value_list, all=True) valid_cut &= validateCut(my_debug_value_list, task, all=True) if 'pcf' in what_to_compare: validatePcf(my_debug_value_list, task, all=True, silent=False) if 'relevance' in what_to_compare: validateRelevanceAnalysis(sas_task, translationkey, my_h, timeout=timeout) return ProblemResults(problemfile, times, samehmax, samegoalzone, valid_cut)